{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") rxcui_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxcui","en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, rxcui_resolver]) data = spark.createDataFrame([["He was seen by the endocrinology service and she was discharged on 50 mg of eltrombopag oral at night, 5 mg amlodipine with meals, and metformin 1000 mg two times a day"]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala ... val chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val rxcui_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxcui","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, rxcui_resolver)) val data = Seq("He was seen by the endocrinology service and she was discharged on 50 mg of eltrombopag oral at night, 5 mg amlodipine with meals, and metformin 1000 mg two times a day").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +---------------------------+--------+-----------------------------------------------------+ | chunk | code | term | +---------------------------+--------+-----------------------------------------------------+ | 50 mg of eltrombopag oral | 825427 | eltrombopag 50 MG Oral Tablet | | 5 mg amlodipine | 197361 | amlodipine 5 MG Oral Tablet | | metformin 1000 mg | 861004 | metformin hydrochloride 2000 MG Oral Tablet | +---------------------------+--------+-----------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_rxcui | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.5 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on November 2020 RxNorm Clinical Drugs ontology graph with ``sbiobert_base_cased_mli`` embeddings. https://www.nlm.nih.gov/pubs/techbull/nd20/brief/nd20_rx_norm_november_release.html. [Sample Content](https://rxnav.nlm.nih.gov/REST/rxclass/class/byRxcui.json?rxcui=1000000). --- layout: model title: Part of Speech for Turkish author: John Snow Labs name: pos_ud_imst date: 2021-03-08 tags: [part_of_speech, open_source, turkish, pos_ud_imst, tr] task: Part of Speech Tagging language: tr edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADJ - PROPN - PUNCT - ADP - NOUN - VERB - PRON - ADV - NUM - AUX - CCONJ - DET - INTJ - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_imst_tr_3.0.0_3.0_1615230214154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_imst_tr_3.0.0_3.0_1615230214154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_imst", "tr") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([["John Snow Labs'tan merhaba! "]], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_imst", "tr") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("John Snow Labs'tan merhaba! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""John Snow Labs'tan merhaba! ""] token_df = nlu.load('tr.pos.ud_imst').predict(text) token_df ```

## Results ```bash token pos 0 John NOUN 1 Snow PROPN 2 Labs'tan PROPN 3 merhaba NOUN 4 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_imst| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|tr| --- layout: model title: English asr_wav2vec2_large_960h_lv60_self TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_large_960h_lv60_self date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h_lv60_self` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_960h_lv60_self_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_self_en_4.2.0_3.0_1664037014758.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_self_en_4.2.0_3.0_1664037014758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_960h_lv60_self', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_960h_lv60_self", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_960h_lv60_self| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.3 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_final_784824218 date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-final-784824218` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824218_en_4.3.1_3.0_1677881805603.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824218_en_4.3.1_3.0_1677881805603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824218","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824218","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_final_784824218| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-final-784824218 --- layout: model title: Part of Speech for Romanian author: John Snow Labs name: pos_ud_rrt date: 2020-05-04 23:32:00 +0800 task: Part of Speech Tagging language: ro edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, ro] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_rrt_ro_2.5.0_2.4_1588622539956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_rrt_ro_2.5.0_2.4_1588622539956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_rrt", "ro") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_rrt", "ro") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale."""] pos_df = nlu.load('ro.pos.ud_rrt').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=1, result='ADP', metadata={'word': 'În'}), Row(annotatorType='pos', begin=3, end=7, result='ADV', metadata={'word': 'afară'}), Row(annotatorType='pos', begin=9, end=10, result='ADP', metadata={'word': 'de'}), Row(annotatorType='pos', begin=12, end=12, result='PART', metadata={'word': 'a'}), Row(annotatorType='pos', begin=14, end=15, result='AUX', metadata={'word': 'fi'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_rrt| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|ro| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_rbt4_h312 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rbt4-h312` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_h312_zh_4.2.4_3.0_1670327101139.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_h312_zh_4.2.4_3.0_1670327101139.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt4_h312","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt4_h312","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt4_h312| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|43.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt4-h312 - https://github.com/iflytek/MiniRBT - https://github.com/ymcui/LERT - https://github.com/ymcui/PERT - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/iflytek/HFL-Anthology --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from nlpunibo) author: John Snow Labs name: distilbert_qa_base_config1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert_base_config1` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config1_en_4.3.0_3.0_1672774413705.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config1_en_4.3.0_3.0_1672774413705.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_config1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/distilbert_base_config1 --- layout: model title: Extract Entities in Spanish Clinical Trial Abstracts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_clinical_trials_abstracts date: 2022-08-11 tags: [es, clinical, licensed, token_classification, bert, ner] task: Named Entity Recognition language: es edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Named Entity Recognition model is intended for detecting relevant entities from Spanish clinical trial abstracts and trained using the BertForTokenClassification method from the transformers library and [BERT based](https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased) embeddings. The model detects Pharmacological and Chemical Substances (CHEM), pathologies (DISO), and lab tests, diagnostic or therapeutic procedures (PROC). ## Predicted Entities `CHEM`, `DISO`, `PROC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_es_4.0.2_3.0_1660229117151.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_es_4.0.2_3.0_1660229117151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_clinical_trials_abstracts", "es", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("label")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["sentence","token","label"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame(["""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales."""], StringType()).toDF("text") result = model.transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_clinical_trials_abstracts", "es", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("label") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("sentence","token","label")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter)) val data = Seq(Array("""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.classify.bert_token.clinical_trials_abstract").predict("""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales.""") ```

## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |suplementación |PROC | |ácido fólico |CHEM | |niveles de homocisteína|PROC | |hemodiálisis |PROC | |hiperhomocisteinemia |DISO | |niveles de homocisteína|PROC | |tHcy |PROC | |ácido fólico |CHEM | |vitamina B6 |CHEM | |pp |CHEM | |diálisis |PROC | |función residual |PROC | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_clinical_trials_abstracts| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References The model is prepared using the reference paper: "A clinical trials corpus annotated with UMLS entities to enhance the access to evidence-based medicine", Leonardo Campillos-Llanos, Ana Valverde-Mateos, Adrián Capllonch-Carrión and Antonio Moreno-Sandoval. BMC Medical Informatics and Decision Making volume 21, Article number: 69 (2021) ## Benchmarking ```bash label precision recall f1-score support B-CHEM 0.9335 0.9314 0.9325 4944 I-CHEM 0.8210 0.8689 0.8443 1251 B-DISO 0.9406 0.9429 0.9417 5538 I-DISO 0.9071 0.9115 0.9093 5129 B-PROC 0.8850 0.9113 0.8979 5893 I-PROC 0.8711 0.8615 0.8663 7047 micro-avg 0.9010 0.9070 0.9040 29802 macro-avg 0.8930 0.9046 0.8987 29802 weighted-avg 0.9012 0.9070 0.9040 29802 ``` --- layout: model title: Estonian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_estonian_legal date: 2023-02-16 tags: [et, estonian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: et edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-estonian-roberta-base` is a Estonian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_estonian_legal_et_4.2.4_3.0_1676577830758.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_estonian_legal_et_4.2.4_3.0_1676577830758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_estonian_legal", "et")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_estonian_legal", "et") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_estonian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|et| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-estonian-roberta-base --- layout: model title: NER Pipeline for Anatomy Entities - Voice of the Patient author: John Snow Labs name: ner_vop_anatomy_pipeline date: 2023-06-09 tags: [licensed, ner, en, pipeline, vop, anatomy] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline extracts mentions of anatomical sites from health-related text in colloquial language. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_pipeline_en_4.4.3_3.0_1686341261132.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_pipeline_en_4.4.3_3.0_1686341261132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_vop_anatomy_pipeline", "en", "clinical/models") pipeline.annotate(" Ugh, I pulled a muscle in my neck from sleeping weird last night. It's like a knot in my trapezius and it hurts to turn my head. ") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_vop_anatomy_pipeline", "en", "clinical/models") val result = pipeline.annotate(" Ugh, I pulled a muscle in my neck from sleeping weird last night. It's like a knot in my trapezius and it hurts to turn my head. ") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_samantharhay', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_samantharhay", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_samantharhay| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.8 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from niklaspm) author: John Snow Labs name: bert_qa_linkbert_large_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `linkbert-large-finetuned-squad` is a English model orginally trained by `niklaspm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_linkbert_large_finetuned_squad_en_4.0.0_3.0_1654188104988.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_linkbert_large_finetuned_squad_en_4.0.0_3.0_1654188104988.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_linkbert_large_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_linkbert_large_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.link_bert.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_linkbert_large_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/niklaspm/linkbert-large-finetuned-squad - https://arxiv.org/abs/2203.15827 --- layout: model title: Translate Catalan to English Pipeline author: John Snow Labs name: translate_ca_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ca, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ca` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ca_en_xx_2.7.0_2.4_1609691497747.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ca_en_xx_2.7.0_2.4_1609691497747.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ca_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ca_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ca.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ca_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: RCT Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_rct_biobert date: 2022-03-01 tags: [licensed, sequence_classification, bert, en, rct] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT-based](https://github.com/dmis-lab/biobert) classifier that can classify the sections within the abstracts of scientific articles regarding randomized clinical trials (RCT). ## Predicted Entities `BACKGROUND`, `CONCLUSIONS`, `METHODS`, `OBJECTIVE`, `RESULTS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_rct_biobert_en_3.4.1_2.4_1646129655723.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_rct_biobert_en_3.4.1_2.4_1646129655723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ |text |class | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ |[Previous attempts to prevent all the unwanted postoperative responses to major surgery with an epidural hydrophilic opioid , morphine , have not succeeded . The authors ' hypothesis was that the lipophilic opioid fentanyl , infused epidurally close to the spinal-cord opioid receptors corresponding to the dermatome of the surgical incision , gives equal pain relief but attenuates postoperative hormonal and metabolic responses more effectively than does systemic fentanyl .]|[BACKGROUND]| +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_rct_biobert| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://arxiv.org/abs/1710.06071 ## Benchmarking ```bash label precision recall f1-score support BACKGROUND 0.77 0.86 0.81 2000 CONCLUSIONS 0.96 0.95 0.95 2000 METHODS 0.96 0.98 0.97 2000 OBJECTIVE 0.85 0.77 0.81 2000 RESULTS 0.98 0.95 0.96 2000 accuracy 0.9 0.9 0.9 10000 macro-avg 0.9 0.9 0.9 10000 weighted-avg 0.9 0.9 0.9 10000 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from SauravMaheshkar) author: John Snow Labs name: xlm_roberta_qa_xlm_multi_roberta_large_chaii date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-multi-roberta-large-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_multi_roberta_large_chaii_en_4.0.0_3.0_1655988703987.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_multi_roberta_large_chaii_en_4.0.0_3.0_1655988703987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_multi_roberta_large_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_multi_roberta_large_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.xlm_roberta.large_multi.by_SauravMaheshkar").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_multi_roberta_large_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/xlm-multi-roberta-large-chaii --- layout: model title: Legal Sub Advisory Agreement Document Binary Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_sub_advisory_agreement_bert date: 2022-12-18 tags: [en, legal, classification, licensed, document, bert, sub, advisory, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_sub_advisory_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `sub-advisory-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `sub-advisory-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sub_advisory_agreement_bert_en_1.0.0_3.0_1671393859780.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sub_advisory_agreement_bert_en_1.0.0_3.0_1671393859780.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_sub_advisory_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[sub-advisory-agreement]| |[other]| |[other]| |[sub-advisory-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sub_advisory_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.99 0.99 204 sub-advisory-agreement 0.98 0.98 0.98 107 accuracy - - 0.99 311 macro-avg 0.99 0.99 0.99 311 weighted-avg 0.99 0.99 0.99 311 ``` --- layout: model title: Recognize Entities OntoNotes - ELECTRA Large author: John Snow Labs name: onto_recognize_entities_electra_large date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [en, open_source, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `electra_large_uncased` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_large_en_2.7.0_2.4_1607530726468.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_large_en_2.7.0_2.4_1607530726468.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_electra_large') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_electra_large") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.onto.large").predict("""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.""") ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_electra_large| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: Part of Speech for Dutch author: John Snow Labs name: pos_ud_alpino date: 2021-03-08 tags: [part_of_speech, open_source, dutch, pos_ud_alpino, nl] task: Part of Speech Tagging language: nl edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - PRON - AUX - ADV - VERB - PUNCT - ADP - NUM - NOUN - SCONJ - DET - ADJ - PROPN - CCONJ - SYM - X - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_alpino_nl_3.0.0_3.0_1615230249057.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_alpino_nl_3.0.0_3.0_1615230249057.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_alpino", "nl") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Hallo van John Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_alpino", "nl") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hallo van John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Hallo van John Snow Labs! ""] token_df = nlu.load('nl.pos.ud_alpino').predict(text) token_df ```

## Results ```bash token pos 0 Hallo PROPN 1 van ADP 2 John PROPN 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_alpino| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|nl| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_8_h_768 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-8_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_768_zh_4.2.4_3.0_1670021813941.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_768_zh_4.2.4_3.0_1670021813941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_8_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|277.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-8_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: English BertForQuestionAnswering Cased model (from Callmenicky) author: John Snow Labs name: bert_qa_callmenicky_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `Callmenicky`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_callmenicky_finetuned_squad_en_4.0.0_3.0_1657185991318.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_callmenicky_finetuned_squad_en_4.0.0_3.0_1657185991318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_callmenicky_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_callmenicky_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_callmenicky_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Callmenicky/bert-finetuned-squad --- layout: model title: Pipeline to Detect Organism in Medical Texts author: John Snow Labs name: bert_token_classifier_ner_linnaeus_species_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_linnaeus_species](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_linnaeus_species_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_linnaeus_species_pipeline_en_4.3.0_3.2_1679303734578.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_linnaeus_species_pipeline_en_4.3.0_3.2_1679303734578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_linnaeus_species_pipeline", "en", "clinical/models") text = '''First identified in chicken, vigilin homologues have now been found in human (6), Xenopus laevis (7), Drosophila melanogaster (8) and Schizosaccharomyces pombe.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_linnaeus_species_pipeline", "en", "clinical/models") val text = "First identified in chicken, vigilin homologues have now been found in human (6), Xenopus laevis (7), Drosophila melanogaster (8) and Schizosaccharomyces pombe." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------------|--------:|------:|:------------|-------------:| | 0 | chicken | 20 | 26 | SPECIES | 0.998697 | | 1 | human | 71 | 75 | SPECIES | 0.999767 | | 2 | Xenopus laevis | 82 | 95 | SPECIES | 0.999918 | | 3 | Drosophila melanogaster | 102 | 124 | SPECIES | 0.999925 | | 4 | Schizosaccharomyces pombe | 134 | 158 | SPECIES | 0.999881 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_linnaeus_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English BertForQuestionAnswering model (from clagator) author: John Snow Labs name: bert_qa_biobert_squad2_cased date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert_squad2_cased` is a English model orginally trained by `clagator`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_squad2_cased_en_4.0.0_3.0_1654185692636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_squad2_cased_en_4.0.0_3.0_1654185692636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_squad2_cased","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_squad2_cased","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.biobert.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_squad2_cased| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/clagator/biobert_squad2_cased --- layout: model title: Sundanese asr_wav2vec2_large_xlsr_sundanese TFWav2Vec2ForCTC from cahya author: John Snow Labs name: asr_wav2vec2_large_xlsr_sundanese date: 2022-09-24 tags: [wav2vec2, su, audio, open_source, asr] task: Automatic Speech Recognition language: su edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_sundanese` is a Sundanese model originally trained by cahya. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_sundanese_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_sundanese_su_4.2.0_3.0_1664039112850.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_sundanese_su_4.2.0_3.0_1664039112850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_sundanese", "su")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_sundanese", "su") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_sundanese| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|su| |Size:|1.2 GB| --- layout: model title: French Bert Embeddings author: John Snow Labs name: bert_embeddings_bert_base_fr_cased date: 2022-04-11 tags: [bert, embeddings, fr, open_source] task: Embeddings language: fr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-fr-cased` is a French model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_fr_cased_fr_3.4.2_3.0_1649675673587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_fr_cased_fr_3.4.2_3.0_1649675673587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_fr_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fr| |Size:|393.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-fr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate Latvian to English Pipeline author: John Snow Labs name: translate_lv_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, lv, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `lv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_lv_en_xx_2.7.0_2.4_1609690492134.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_lv_en_xx_2.7.0_2.4_1609690492134.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_lv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_lv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.lv.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_lv_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Clinical Entities author: John Snow Labs name: ner_jsl_greedy_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_greedy_biobert](https://nlp.johnsnowlabs.com/2021/08/13/ner_jsl_greedy_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_pipeline_en_3.4.1_3.0_1647869992577.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_pipeline_en_3.4.1_3.0_1647869992577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_jsl_greedy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("ner_jsl_greedy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biobert_jsl_greedy.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_xdoc_base_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_xdoc_base_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_xdoc_base_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/microsoft/xdoc-base-squad2.0 - https://arxiv.org/abs/2210.02849 --- layout: model title: English image_classifier_vit_Check_Missing_Teeth ViTForImageClassification from steven123 author: John Snow Labs name: image_classifier_vit_Check_Missing_Teeth date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Check_Missing_Teeth` is a English model originally trained by steven123. ## Predicted Entities `Missing Teeth`, `Non-Missing Teeth` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Check_Missing_Teeth_en_4.1.0_3.0_1660167758083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Check_Missing_Teeth_en_4.1.0_3.0_1660167758083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Check_Missing_Teeth", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Check_Missing_Teeth", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Check_Missing_Teeth| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from Luvale to English author: John Snow Labs name: opus_mt_lue_en date: 2020-12-29 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, lue, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `lue` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_lue_en_xx_2.7.0_2.4_1609254447650.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_lue_en_xx_2.7.0_2.4_1609254447650.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_lue_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Persian BertForQuestionAnswering Base Uncased model (from mhmsadegh) author: John Snow Labs name: bert_qa_base_parsbert_uncased_finetuned_squad date: 2022-07-07 tags: [fa, open_source, bert, question_answering] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-parsbert-uncased-finetuned-squad` is a Persian model originally trained by `mhmsadegh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_parsbert_uncased_finetuned_squad_fa_4.0.0_3.0_1657183402278.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_parsbert_uncased_finetuned_squad_fa_4.0.0_3.0_1657183402278.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_parsbert_uncased_finetuned_squad","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_parsbert_uncased_finetuned_squad","fa") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_parsbert_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/mhmsadegh/bert-base-parsbert-uncased-finetuned-squad --- layout: model title: Legal Modifications Clause Binary Classifier author: John Snow Labs name: legclf_modifications_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `modifications` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `modifications` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_modifications_clause_en_1.0.0_3.2_1660123743355.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_modifications_clause_en_1.0.0_3.2_1660123743355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_modifications_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[modifications]| |[other]| |[other]| |[modifications]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_modifications_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support modifications 0.89 0.83 0.86 76 other 0.92 0.95 0.94 168 accuracy - - 0.91 244 macro-avg 0.91 0.89 0.90 244 weighted-avg 0.91 0.91 0.91 244 ``` --- layout: model title: English BERT Embeddings Cased model (from mrm8488) author: John Snow Labs name: bert_embeddings_bioclinicalbert_finetuned_covid_papers date: 2022-07-15 tags: [en, open_source, bert, embeddings] task: Embeddings language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BERT Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bioclinicalBERT-finetuned-covid-papers` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bioclinicalbert_finetuned_covid_papers_en_4.0.0_3.0_1657880858798.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bioclinicalbert_finetuned_covid_papers_en_4.0.0_3.0_1657880858798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_bioclinicalbert_finetuned_covid_papers","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_bioclinicalbert_finetuned_covid_papers","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bioclinicalbert_finetuned_covid_papers| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|406.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/mrm8488/bioclinicalBERT-finetuned-covid-papers --- layout: model title: Detect Assertion Status from Response to Treatment author: John Snow Labs name: assertion_oncology_response_to_treatment_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of entities related to response to treatment. The model identifies positive mentions (Present_Or_Past status), and hypothetical or absent mentions (Hypothetical_Or_Absent status). ## Predicted Entities `Hypothetical_Or_Absent`, `Present_Or_Past` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_response_to_treatment_wip_en_4.1.0_3.0_1664641698152.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_response_to_treatment_wip_en_4.1.0_3.0_1664641698152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:-----------|:----------------------|:-----------------------| | recurrence | Response_To_Treatment | Hypothetical_Or_Absent | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_response_to_treatment_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Hypothetical_Or_Absent 0.83 0.96 0.89 46.0 Present_Or_Past 0.94 0.79 0.86 43.0 macro-avg 0.89 0.87 0.87 89.0 weighted-avg 0.89 0.88 0.88 89.0 ``` --- layout: model title: English BertForQuestionAnswering model (from ZYW) author: John Snow Labs name: bert_qa_squad_mbert_model date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-mbert-model` is a English model orginally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad_mbert_model_en_4.0.0_3.0_1654192040161.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad_mbert_model_en_4.0.0_3.0_1654192040161.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad_mbert_model","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad_mbert_model","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.multi_lingual_bert.by_ZYW").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad_mbert_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-mbert-model --- layout: model title: Sentence Detection in Tamil Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [ta, open_source, sentence_detection] task: Sentence Detection language: ta edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ta_3.2.0_3.0_1630337465197.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ta_3.2.0_3.0_1630337465197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "ta") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""ஆங்கில வாசிப்பு பத்திகளின் சிறந்த ஆதாரத்தைத் தேடுகிறீர்களா? நீங்கள் சரியான இடத்திற்கு வந்துவிட்டீர்கள். சமீபத்திய ஆய்வின்படி, இன்றைய இளைஞர்களிடம் படிக்கும் பழக்கம் வேகமாக குறைந்து வருகிறது. கொடுக்கப்பட்ட ஆங்கில வாசிப்பு பத்தியில் சில வினாடிகளுக்கு மேல் அவர்களால் கவனம் செலுத்த முடியாது! மேலும், அனைத்து போட்டித் தேர்வுகளிலும் வாசிப்பு ஒரு ஒருங்கிணைந்த பகுதியாகும். எனவே, உங்கள் வாசிப்புத் திறனை எவ்வாறு மேம்படுத்துவது? இந்த கேள்விக்கான பதில் உண்மையில் மற்றொரு கேள்வி: வாசிப்பு திறனின் பயன் என்ன? வாசிப்பின் முக்கிய நோக்கம் 'உணர்த்துவது'.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "ta") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("ஆங்கில வாசிப்பு பத்திகளின் சிறந்த ஆதாரத்தைத் தேடுகிறீர்களா? நீங்கள் சரியான இடத்திற்கு வந்துவிட்டீர்கள். சமீபத்திய ஆய்வின்படி, இன்றைய இளைஞர்களிடம் படிக்கும் பழக்கம் வேகமாக குறைந்து வருகிறது. கொடுக்கப்பட்ட ஆங்கில வாசிப்பு பத்தியில் சில வினாடிகளுக்கு மேல் அவர்களால் கவனம் செலுத்த முடியாது! மேலும், அனைத்து போட்டித் தேர்வுகளிலும் வாசிப்பு ஒரு ஒருங்கிணைந்த பகுதியாகும். எனவே, உங்கள் வாசிப்புத் திறனை எவ்வாறு மேம்படுத்துவது? இந்த கேள்விக்கான பதில் உண்மையில் மற்றொரு கேள்வி: வாசிப்பு திறனின் பயன் என்ன? வாசிப்பின் முக்கிய நோக்கம் 'உணர்த்துவது'.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load('ta.sentence_detector').predict("ஆங்கில வாசிப்பு பத்திகளின் சிறந்த ஆதாரத்தைத் தேடுகிறீர்களா? நீங்கள் சரியான இடத்திற்கு வந்துவிட்டீர்கள். சமீபத்திய ஆய்வின்படி, இன்றைய இளைஞர்களிடம் படிக்கும் பழக்கம் வேகமாக குறைந்து வருகிறது. கொடுக்கப்பட்ட ஆங்கில வாசிப்பு பத்தியில் சில வினாடிகளுக்கு மேல் அவர்களால் கவனம் செலுத்த முடியாது! மேலும், அனைத்து போட்டித் தேர்வுகளிலும் வாசிப்பு ஒரு ஒருங்கிணைந்த பகுதியாகும். எனவே, உங்கள் வாசிப்புத் திறனை எவ்வாறு மேம்படுத்துவது? இந்த கேள்விக்கான பதில் உண்மையில் மற்றொரு கேள்வி: வாசிப்பு திறனின் பயன் என்ன? வாசிப்பின் முக்கிய நோக்கம் 'உணர்த்துவது'.", output_level ='sentence') ```

## Results ```bash +--------------------------------------------------------------------------------------------------+ |result | +--------------------------------------------------------------------------------------------------+ |[ஆங்கில வாசிப்பு பத்திகளின் சிறந்த ஆதாரத்தைத் தேடுகிறீர்களா?] | |[நீங்கள் சரியான இடத்திற்கு வந்துவிட்டீர்கள்.] | |[சமீபத்திய ஆய்வின்படி, இன்றைய இளைஞர்களிடம் படிக்கும் பழக்கம் வேகமாக குறைந்து வருகிறது.] | |[கொடுக்கப்பட்ட ஆங்கில வாசிப்பு பத்தியில் சில வினாடிகளுக்கு மேல் அவர்களால் கவனம் செலுத்த முடியாது!]| |[மேலும், அனைத்து போட்டித் தேர்வுகளிலும் வாசிப்பு ஒரு ஒருங்கிணைந்த பகுதியாகும்.] | |[எனவே, உங்கள் வாசிப்புத் திறனை எவ்வாறு மேம்படுத்துவது?] | |[இந்த கேள்விக்கான பதில் உண்மையில் மற்றொரு கேள்வி:] | |[வாசிப்பு திறனின் பயன் என்ன?] | |[வாசிப்பின் முக்கிய நோக்கம் 'உணர்த்துவது'.] | +--------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|ta| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English BertForSequenceClassification Mini Cased model (from mrm8488) author: John Snow Labs name: bert_sequence_classifier_mini_finetuned_age_news_classification date: 2022-07-13 tags: [en, open_source, bert, sequence_classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-mini-finetuned-age_news-classification` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_mini_finetuned_age_news_classification_en_4.0.0_3.0_1657720835247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_mini_finetuned_age_news_classification_en_4.0.0_3.0_1657720835247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = BertForSequenceClassification.pretrained("bert_sequence_classifier_mini_finetuned_age_news_classification","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = BertForSequenceClassification.pretrained("bert_sequence_classifier_mini_finetuned_age_news_classification","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_mini_finetuned_age_news_classification| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|42.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/bert-mini-finetuned-age_news-classification --- layout: model title: English LongformerForQuestionAnswering model (from manishiitg) Version-2 author: John Snow Labs name: longformer_qa_recruit_v2 date: 2022-06-26 tags: [en, open_source, longformer, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `longformer-recruit-qa-v2` is a English model originally trained by `manishiitg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_qa_recruit_v2_en_4.0.0_3.0_1656255752690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_qa_recruit_v2_en_4.0.0_3.0_1656255752690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_qa_recruit_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_qa_recruit_v2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.longformer.v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_qa_recruit_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|556.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/manishiitg/longformer-recruit-qa-v2 --- layout: model title: Fast Neural Machine Translation Model from English to Finnish author: John Snow Labs name: opus_mt_en_fi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, fi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `fi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_fi_xx_2.7.0_2.4_1609167722467.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_fi_xx_2.7.0_2.4_1609167722467.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_fi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for Snomed Concepts, Body Structure Version (sbiobert_base_cased_mli) author: John Snow Labs name: sbiobertresolve_snomed_bodyStructure date: 2021-07-08 tags: [snomed, en, clinical, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical (anatomical structures) entities to Snomed codes (body structure version) using sentence embeddings. ## Predicted Entities Snomed Codes and their normalized definition with `sbiobert_base_cased_mli` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_bodyStructure_en_3.1.0_2.4_1625732176926.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_bodyStructure_en_3.1.0_2.4_1625732176926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_snomed_bodyStructure``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Disease_Syndrome_Disorder, External_body_part_or_region``` set in ```.setWhiteList()```. ```sbiobertresolve_snomed_bodyStructure``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. No need to set ```.setWhiteList()```. Merge ner_jsl and ner_anatomy_coarse model chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") jsl_sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_snomed_bodyStructure", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("snomed_code") snomed_pipelineModel = PipelineModel( stages = [ documentAssembler, jsl_sbert_embedder, snomed_resolver]) snomed_lp = LightPipeline(snomed_pipelineModel) result = snomed_lp.fullAnnotate("Amputation stump") ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_bodyStructure", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("snomed_code") val snomed_pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, snomed_resolver)) val snomed_lp = LightPipeline(snomed_pipelineModel) val result = snomed_lp.fullAnnotate("Amputation stump") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed_body_structure").predict("""Amputation stump""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:-----------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------| | 0 | amputation stump | 38033009 | [Amputation stump, Amputation stump of upper limb, Amputation stump of left upper limb, Amputation stump of lower limb, Amputation stump of left lower limb, Amputation stump of right upper limb, Amputation stump of right lower limb, ...]| ['38033009', '771359009', '771364008', '771358001', '771367001', '771365009', '771368006', ...] | ['0.0000', '0.0773', '0.0858', '0.0863', '0.0905', '0.0911', '0.0972', ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_bodyStructure| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[snomed_code]| |Language:|en| |Case sensitive:|true| ## Data Source https://www.snomed.org/ --- layout: model title: Arabic Bert Embeddings (from bashar-talafha) author: John Snow Labs name: bert_embeddings_multi_dialect_bert_base_arabic date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `multi-dialect-bert-base-arabic` is a Arabic model orginally trained by `bashar-talafha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_multi_dialect_bert_base_arabic_ar_3.4.2_3.0_1649677978634.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_multi_dialect_bert_base_arabic_ar_3.4.2_3.0_1649677978634.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_multi_dialect_bert_base_arabic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|414.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/bashar-talafha/multi-dialect-bert-base-arabic - https://ai.mawdoo3.com/ - https://github.com/alisafaya/Arabic-BERT - https://sites.google.com/view/nadi-shared-task - https://github.com/mawdoo3/Multi-dialect-Arabic-BERT --- layout: model title: English DistilBertForQuestionAnswering model (from caiosantillo) author: John Snow Labs name: distilbert_qa_caiosantillo_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `caiosantillo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_caiosantillo_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725142690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_caiosantillo_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725142690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_caiosantillo_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_caiosantillo_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_caiosantillo").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_caiosantillo_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/caiosantillo/distilbert-base-uncased-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from Icelandic to English author: John Snow Labs name: opus_mt_is_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, is, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `is` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_is_en_xx_2.7.0_2.4_1609167041296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_is_en_xx_2.7.0_2.4_1609167041296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_is_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from bhavikardeshna) author: John Snow Labs name: bert_qa_multilingual_bert_base_cased_english date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `multilingual-bert-base-cased-english` is a English model orginally trained by `bhavikardeshna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_multilingual_bert_base_cased_english_en_4.0.0_3.0_1654188464581.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_multilingual_bert_base_cased_english_en_4.0.0_3.0_1654188464581.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_multilingual_bert_base_cased_english","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_multilingual_bert_base_cased_english","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.multilingual_english_tuned_base_cased.by_bhavikardeshna").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_multilingual_bert_base_cased_english| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bhavikardeshna/multilingual-bert-base-cased-english --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_fpdm_pert_sent_0.01_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fpdm_roberta_pert_sent_0.01_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_pert_sent_0.01_squad2.0_en_4.3.0_3.0_1674211060807.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_pert_sent_0.01_squad2.0_en_4.3.0_3.0_1674211060807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_pert_sent_0.01_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_pert_sent_0.01_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fpdm_pert_sent_0.01_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/fpdm_roberta_pert_sent_0.01_squad2.0 --- layout: model title: Fast Neural Machine Translation Model from English to Xhosa author: John Snow Labs name: opus_mt_en_xh date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, xh, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `xh` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_xh_xx_2.7.0_2.4_1609163416968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_xh_xx_2.7.0_2.4_1609163416968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_xh| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_dm128 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-dm128` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dm128_en_4.3.0_3.0_1675118965696.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dm128_en_4.3.0_3.0_1675118965696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_dm128","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_dm128","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_dm128| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|37.4 MB| ## References - https://huggingface.co/google/t5-efficient-small-dm128 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English BertForQuestionAnswering model (from AnonymousSub) author: John Snow Labs name: bert_qa_fpdm_triplet_bert_FT_newsqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fpdm_triplet_bert_FT_newsqa` is a English model orginally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_fpdm_triplet_bert_FT_newsqa_en_4.0.0_3.0_1654187915859.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_fpdm_triplet_bert_FT_newsqa_en_4.0.0_3.0_1654187915859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_fpdm_triplet_bert_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_fpdm_triplet_bert_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.bert.qa_fpdm_triplet_ft.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_fpdm_triplet_bert_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/fpdm_triplet_bert_FT_newsqa --- layout: model title: Detect Drug Information author: John Snow Labs name: ner_posology_en date: 2020-04-15 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [ner, en, licensed, clinical] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Pretrained named entity recognition deep learning model for posology. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. {:.h2_title} ## Predicted Entities ``DOSAGE``, ``DRUG``, ``DURATION``, ``FORM``, ``FREQUENCY``, ``ROUTE``, ``STRENGTH``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_en_2.4.4_2.4_1584452534235.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_en_2.4.4_2.4_1584452534235.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPython.html %} ```python ... embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_posology", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.']], ["text"])) ``` ```scala ... val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_posology", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_nso", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_nso", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.nso').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_nso| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from andi611) Squad2 with Neg, Multi, Repeat author: John Snow Labs name: distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-squad2-with-ner-with-neg-with-multi-with-repeat` is a English model originally trained by `andi611`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat_en_4.0.0_3.0_1654727440196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat_en_4.0.0_3.0_1654727440196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_conll.distil_bert.base_uncased_with_neg_with_multi_with_repeat.by_andi611").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_squad2_with_ner_with_neg_with_multi_with_repeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/andi611/distilbert-base-uncased-squad2-with-ner-with-neg-with-multi-with-repeat --- layout: model title: English DistilBertForQuestionAnswering Small model (from ncduy) author: John Snow Labs name: distilbert_qa_base_cased_distilled_squad_finetuned_squad_small date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-finetuned-squad-small` is a English model originally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_squad_finetuned_squad_small_en_4.0.0_3.0_1654723606084.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_squad_finetuned_squad_small_en_4.0.0_3.0_1654723606084.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_squad_finetuned_squad_small","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_squad_finetuned_squad_small","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_small_cased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_distilled_squad_finetuned_squad_small| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/distilbert-base-cased-distilled-squad-finetuned-squad-small --- layout: model title: Multilabel Classification of Customer Service (Linguistic features) author: John Snow Labs name: finmulticlf_customer_service_lin_features date: 2023-02-03 tags: [en, licensed, finance, classification, customer, linguistic, tensorflow] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multilabel Text Classification model that can help you classify a chat message from customer service according to linguistic features. The classes are the following: - Q - Colloquial variation - P - Politeness variation - W - Offensive language - K - Keyword language - B - Basic syntactic structure - C - Coordinated syntactic structure - I - Interrogative structure - M - Morphological variation (plurals, tenses…) - L - Lexical variation (synonyms) - E - Expanded abbreviations (I'm -> I am, I'd -> I would…) - N - Negation - Z - Noise phenomena like spelling or punctuation errors ## Predicted Entities `B`, `C`, `E`, `I`, `K`, `L`, `M`, `N`, `P`, `Q`, `W`, `Z` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmulticlf_customer_service_lin_features_en_1.0.0_3.0_1675430237309.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmulticlf_customer_service_lin_features_en_1.0.0_3.0_1675430237309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") embeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.MultiClassifierDLModel().load("finmulticlf_customer_service_lin_features", "en", "finance/models")\ .setInputCols("sentence_embeddings") \ .setOutputCol("class") pipeline = nlp.Pipeline().setStages( [ document_assembler, embeddings, docClassifier ] ) empty_data = spark.createDataFrame([[""]]).toDF("text") model = pipeline.fit(empty_data) light_model = nlp.LightPipeline(model) result = light_model.annotate("""What do i have to ddo to cancel a Gold account""") result["class"] ```

## Results ```bash ['Q', 'B', 'L', 'Z', 'I'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmulticlf_customer_service_lin_features| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|13.0 MB| ## References https://github.com/bitext/customer-support-intent-detection-training-dataset ## Benchmarking ```bash label precision recall f1-score support B 1.00 1.00 1.00 485 C 0.79 0.80 0.80 61 E 0.74 0.89 0.80 44 I 0.95 0.94 0.94 134 K 0.96 0.96 0.96 108 L 0.96 0.97 0.96 402 M 0.93 0.93 0.93 134 N 0.90 0.75 0.82 12 P 0.77 0.90 0.83 30 Q 0.73 0.68 0.71 212 W 0.85 0.88 0.87 33 Z 0.68 0.72 0.70 160 micro-avg 0.90 0.90 0.90 1815 macro-avg 0.85 0.87 0.86 1815 weighted-avg 0.90 0.90 0.90 1815 samples-avg 0.91 0.92 0.90 1815 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Indonesian author: John Snow Labs name: opus_mt_en_id date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, id, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `id` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_id_xx_2.7.0_2.4_1609164704175.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_id_xx_2.7.0_2.4_1609164704175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_id| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Stopwords Remover for Norwegian Bokmål language (211 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, nb, open_source] task: Stop Words Removal language: nb edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_nb_3.4.1_3.0_1646673078011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_nb_3.4.1_3.0_1646673078011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_xdoc_base_squad1.1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_xdoc_base_squad1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_xdoc_base_squad1.1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/microsoft/xdoc-base-squad1.1 - https://arxiv.org/abs/2210.02849 --- layout: model title: Malay T5ForConditionalGeneration Tiny Cased model (from mesolitica) author: John Snow Labs name: t5_super_tiny_bahasa_cased date: 2023-01-31 tags: [ms, open_source, t5, tensorflow] task: Text Generation language: ms edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-super-tiny-bahasa-cased` is a Malay model originally trained by `mesolitica`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_super_tiny_bahasa_cased_ms_4.3.0_3.0_1675156057502.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_super_tiny_bahasa_cased_ms_4.3.0_3.0_1675156057502.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_super_tiny_bahasa_cased","ms") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_super_tiny_bahasa_cased","ms") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_super_tiny_bahasa_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ms| |Size:|40.5 MB| ## References - https://huggingface.co/mesolitica/t5-super-tiny-bahasa-cased - https://github.com/huseinzol05/malaya/tree/master/pretrained-model/t5/prepare - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5 --- layout: model title: Legal Arguments Mining in Court Decisions (in German) author: John Snow Labs name: legclf_argument_mining_german date: 2023-03-26 tags: [de, licensed, classification, legal, tensorflow] task: Text Classification language: de edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model in German which classifies arguments in legal discourse. These are the following classes: `subsumption`, `definition`, `conclusion`, `other`. ## Predicted Entities `subsumption`, `definition`, `conclusion`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_argument_mining_german_de_1.0.0_3.0_1679848514704.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_argument_mining_german_de_1.0.0_3.0_1679848514704.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_large_german_legal", "de")\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) embeddingsSentence = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"])\ .setOutputCol("sentence_embeddings")\ .setPoolingStrategy("AVERAGE")\ docClassifier = legal.ClassifierDLModel.pretrained("legclf_argument_mining_de", "de", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, embeddingsSentence, docClassifier ]) df = spark.createDataFrame([["Folglich liegt eine Verletzung von Artikel 8 der Konvention vor ."]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) result.select("text", "category.result").show(truncate=False) ```

## Results ```bash +-----------------------------------------------------------------+------------+ |text |result | +-----------------------------------------------------------------+------------+ |Folglich liegt eine Verletzung von Artikel 8 der Konvention vor .|[conclusion]| +-----------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_argument_mining_german| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|de| |Size:|24.0 MB| ## References Train dataset available [here](https://huggingface.co/datasets/MeilingShi/legal_argument_mining) ## Benchmarking ```bash label precision recall f1-score support conclusion 0.88 0.88 0.88 52 definition 0.83 0.83 0.83 58 other 0.86 0.88 0.87 49 subsumption 0.81 0.80 0.80 64 accuracy - - 0.84 223 macro avg 0.85 0.85 0.85 223 weighted avg 0.84 0.84 0.84 223 ``` --- layout: model title: Match Datetime in Texts author: John Snow Labs name: match_datetime date: 2022-01-04 tags: [en, open_source] task: Pipeline Public language: en nav_key: models edition: Spark NLP 3.3.4 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description DateMatcher based on yyyy/MM/dd {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/1.SparkNLP_Basics.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/match_datetime_en_3.3.4_3.0_1641310187437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/match_datetime_en_3.3.4_3.0_1641310187437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline_local = PretrainedPipeline('match_datetime') tres = pipeline_local.fullAnnotate(input_list)[0] for dte in tres['date']: sent = tres['sentence'][int(dte.metadata['sentence'])] print (f'text/chunk {sent.result[dte.begin:dte.end+1]} | mapped_date: {dte.result}') ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline import com.johnsnowlabs.nlp.SparkNLP SparkNLP.version() val testData = spark.createDataFrame(Seq( (1, "David visited the restaurant yesterday with his family. He also visited and the day before, but at that time he was alone. David again visited today with his colleagues. He and his friends really liked the food and hoped to visit again tomorrow."))).toDF("id", "text") val pipeline = PretrainedPipeline("match_datetime", lang="en") val annotation = pipeline.transform(testData) annotation.show() ```

## Results ```bash text/chunk yesterday | mapped_date: 2022/01/02 text/chunk day before | mapped_date: 2022/01/02 text/chunk today | mapped_date: 2022/01/03 text/chunk tomorrow | mapped_date: 2022/01/04 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|match_datetime| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|12.9 KB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - MultiDateMatcher --- layout: model title: Word2Vec Embeddings in Romansh (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, rm, open_source] task: Embeddings language: rm edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_rm_3.4.1_3.0_1647454087483.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_rm_3.4.1_3.0_1647454087483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("distilroberta_base_token_classifier_ontonotes_pipeline", lang = "en") pipeline.annotate("My name is John and I have been working at John Snow Labs since November 2020.") ``` ```scala val pipeline = new PretrainedPipeline("distilroberta_base_token_classifier_ontonotes_pipeline", lang = "en") pipeline.annotate("My name is John and I have been working at John Snow Labs since November 2020.")) ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PERSON | |John Snow Labs|ORG | |November 2020 |DATE | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilroberta_base_token_classifier_ontonotes_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|307.5 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: Legal Now therefore Clause Binary Classifier author: John Snow Labs name: legclf_now_therefore_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `now-therefore` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `now-therefore` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_now_therefore_clause_en_1.0.0_3.2_1660122766408.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_now_therefore_clause_en_1.0.0_3.2_1660122766408.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_now_therefore_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[now-therefore]| |[other]| |[other]| |[now-therefore]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_now_therefore_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support now-therefore 0.98 0.98 0.98 58 other 0.99 0.99 0.99 146 accuracy - - 0.99 204 macro-avg 0.99 0.99 0.99 204 weighted-avg 0.99 0.99 0.99 204 ``` --- layout: model title: Chamorro RobertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: roberta_qa_ADDI_CH_RoBERTa date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: ch edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-CH-RoBERTa` is a Chamorro model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_CH_RoBERTa_ch_4.0.0_3.0_1655726262986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_CH_RoBERTa_ch_4.0.0_3.0_1655726262986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ADDI_CH_RoBERTa","ch") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ADDI_CH_RoBERTa","ch") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ch.answer_question.roberta.ch_tuned.by_Gantenbein").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ADDI_CH_RoBERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|ch| |Size:|421.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-CH-RoBERTa --- layout: model title: Swedish BERT Sentence Base Cased Embedding author: John Snow Labs name: sent_bert_base_cased date: 2021-09-06 tags: [swedish, bert_sentence_embeddings, open_source, cased, sv] task: Embeddings language: sv edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on aproximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_sv_3.2.2_3.0_1630926268941.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_sv_3.2.2_3.0_1630926268941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "sv") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "sv") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu nlu.load("sv.embed_sentence.bert.base_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_base_cased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|sv| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/KB/bert-base-swedish-cased --- layout: model title: ALBERT Large CoNNL-03 NER Pipeline author: ahmedlone127 name: albert_large_token_classifier_conll03_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, albert, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [albert_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/albert_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/albert_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655211084220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/albert_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655211084220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("albert_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("albert_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.")) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_drugs_greedy_pipeline", "en", "clinical/models") pipeline.annotate("DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.") ``` ```scala val pipeline = new PretrainedPipeline("ner_drugs_greedy_pipeline", "en", "clinical/models") pipeline.annotate("DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.drugs_greedy.pipeline").predict("""DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.""") ```

## Results ```bash +-----------------------------------+------------+ | chunk | ner_label | +-----------------------------------+------------+ | hydrocortisone tablets | DRUG | | 20 mg to 240 mg of hydrocortisone | DRUG | +-----------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_greedy_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Legal Reinstatement Clause Binary Classifier author: John Snow Labs name: legclf_reinstatement_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `reinstatement` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `reinstatement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_reinstatement_clause_en_1.0.0_3.2_1660122901279.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_reinstatement_clause_en_1.0.0_3.2_1660122901279.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_reinstatement_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[reinstatement]| |[other]| |[other]| |[reinstatement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_reinstatement_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.97 0.96 101 reinstatement 0.89 0.83 0.86 29 accuracy - - 0.94 130 macro-avg 0.92 0.90 0.91 130 weighted-avg 0.94 0.94 0.94 130 ``` --- layout: model title: Detect Anatomical Structures (Single Entity - embeddings_clinical) author: John Snow Labs name: ner_anatomy_coarse date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description An NER model to extract all types of anatomical references in text using "embeddings_clinical" embeddings. It is a single entity model and generalizes all anatomical references to a single entity. ## Predicted Entities `Anatomy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ANATOMY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_en_3.0.0_3.0_1617209678971.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_en_3.0.0_3.0_1617209678971.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_anatomy_coarse", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["content in the lung tissue"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_anatomy_coarse", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""content in the lung tissue""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy.coarse").predict("""content in the lung tissue""") ```

## Results ```bash | | ner_chunk | entity | |---:|------------------:|----------:| | 0 | lung tissue | Anatomy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_coarse| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on a custom dataset using 'embeddings_clinical'. ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|:--------------|------:|------:|------:|---------:|---------:|---------:| | 0 | B-Anatomy | 2568 | 165 | 158 | 0.939627 | 0.94204 | 0.940832 | | 1 | I-Anatomy | 1692 | 89 | 169 | 0.950028 | 0.909189 | 0.92916 | | 2 | Macro-average | 4260 | 254 | 327 | 0.944827 | 0.925614 | 0.935122 | | 3 | Micro-average | 4260 | 254 | 327 | 0.943731 | 0.928712 | 0.936161 | ``` --- layout: model title: BERT Sentence Embeddings trained on Wikipedia and BooksCorpus and fine-tuned on QQP author: John Snow Labs name: sent_bert_wiki_books_qqp date: 2021-08-31 tags: [en, sentence_embeddings, open_source, wikipedia_dataset, books_corpus_dataset, qqp_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/wiki_books/1 and fine-tuned on QQP. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. This model is intended to be used for a variety of English NLP tasks. The pre-training data contains more formal text and the model may not generalize to more colloquial text such as social media or messages. This model is fine-tuned on the QQP and is recommended for use in semantic similarity of question pairs tasks. The fine-tuning task uses the Quora Question Pairs (QQP) dataset to predict whether two questions are duplicates or not. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_qqp_en_3.2.0_3.0_1630412104798.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_qqp_en_3.2.0_3.0_1630412104798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_qqp", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_qqp", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.wiki_books_qqp').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_wiki_books_qqp| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [Wikipedia dataset](https://dumps.wikimedia.org/) [2]: [BooksCorpus dataset](http://yknzhu.wixsite.com/mbweb) [3]: [Quora Question Pairs (QQP) dataset](https://www.quora.com/q/quoradata/First-Quora-Dataset-Release-Question-Pairs) This Model has been imported from: https://tfhub.dev/google/experts/bert/wiki_books/qqp/2 --- layout: model title: Portuguese Legal Roberta Embeddings author: John Snow Labs name: roberta_base_portuguese_legal date: 2023-02-16 tags: [pt, portuguese, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: pt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-portuguese-roberta-base` is a Portuguese model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_portuguese_legal_pt_4.2.4_3.0_1676563448655.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_portuguese_legal_pt_4.2.4_3.0_1676563448655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_portuguese_legal", "pt")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_portuguese_legal", "pt") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_portuguese_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|pt| |Size:|415.9 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-portuguese-roberta-base --- layout: model title: Fast Neural Machine Translation Model from Korean to English author: John Snow Labs name: opus_mt_ko_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ko, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ko` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ko_en_xx_2.7.0_2.4_1609168124610.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ko_en_xx_2.7.0_2.4_1609168124610.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ko_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual XLMRobertaForTokenClassification Cased model (from nbroad) author: John Snow Labs name: xlmroberta_ner_jplu_r_40_lang date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `jplu-xlm-r-ner-40-lang` is a Multilingual model originally trained by `nbroad`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jplu_r_40_lang_xx_4.1.0_3.0_1660422549645.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jplu_r_40_lang_xx_4.1.0_3.0_1660422549645.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jplu_r_40_lang","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jplu_r_40_lang","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jplu_r_40_lang| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|967.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nbroad/jplu-xlm-r-ner-40-lang --- layout: model title: English DistilBertForQuestionAnswering Cased model (from aszidon) author: John Snow Labs name: distilbert_qa_custom4 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom4` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom4_en_4.3.0_3.0_1672774680666.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom4_en_4.3.0_3.0_1672774680666.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom4 --- layout: model title: English BertForQuestionAnswering model (from LenaSchmidt) author: John Snow Labs name: bert_qa_no_need_to_name_this date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `no_need_to_name_this` is a English model orginally trained by `LenaSchmidt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_no_need_to_name_this_en_4.0.0_3.0_1654188966070.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_no_need_to_name_this_en_4.0.0_3.0_1654188966070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_no_need_to_name_this","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_no_need_to_name_this","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_no_need_to_name_this| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LenaSchmidt/no_need_to_name_this --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from navteca) author: John Snow Labs name: roberta_qa_navteca_base_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `navteca`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_navteca_base_squad2_en_4.2.4_3.0_1669986777716.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_navteca_base_squad2_en_4.2.4_3.0_1669986777716.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_navteca_base_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_navteca_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_navteca_base_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/navteca/roberta-base-squad2 - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Chinese BertForMaskedLM Mini Cased model (from hfl) author: John Snow Labs name: bert_embeddings_minirbt_h256 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `minirbt-h256` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_minirbt_h256_zh_4.2.4_3.0_1670022628583.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_minirbt_h256_zh_4.2.4_3.0_1670022628583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_minirbt_h256","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_minirbt_h256","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_minirbt_h256| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|39.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/minirbt-h256 - https://github.com/iflytek/MiniRBT - https://github.com/ymcui/LERT - https://github.com/ymcui/PERT - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/iflytek/HFL-Anthology --- layout: model title: English asr_wav2vec2_xls_r_tf_left_right_shuru TFWav2Vec2ForCTC from hrdipto author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_tf_left_right_shuru` is a English model originally trained by hrdipto. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru_en_4.2.0_3.0_1664040053317.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru_en_4.2.0_3.0_1664040053317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_tf_left_right_shuru| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_cased_finetuned_squad_r3f date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-finetuned-squad-r3f` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_cased_finetuned_squad_r3f_en_4.0.0_3.0_1657182921335.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_cased_finetuned_squad_r3f_en_4.0.0_3.0_1657182921335.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_cased_finetuned_squad_r3f","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_cased_finetuned_squad_r3f","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_cased_finetuned_squad_r3f| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-cased-finetuned-squad-r3f --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from kj141) author: John Snow Labs name: distilbert_qa_kj141_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `kj141`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_kj141_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771876070.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_kj141_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771876070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kj141_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kj141_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_kj141_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kj141/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Indemnification and contribution Clause Binary Classifier (md) author: John Snow Labs name: legclf_indemnification_and_contribution_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `indemnification-and-contribution` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `indemnification-and-contribution` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_and_contribution_md_en_1.0.0_3.0_1669376499482.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_and_contribution_md_en_1.0.0_3.0_1669376499482.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_indemnification_and_contribution_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[indemnification-and-contribution]| |[other]| |[other]| |[indemnification-and-contribution]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnification_and_contribution_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support indemnification 0.92 0.96 0.94 25 other 0.97 0.95 0.96 39 accuracy 0.95 64 macro avg 0.95 0.95 0.95 64 weighted avg 0.95 0.95 0.95 64 ``` --- layout: model title: Korean ElectraForQuestionAnswering model (from sehandev) author: John Snow Labs name: electra_qa_long date: 2022-06-22 tags: [ko, open_source, electra, question_answering] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `koelectra-long-qa` is a Korean model originally trained by `sehandev`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_long_ko_4.0.0_3.0_1655922278586.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_long_ko_4.0.0_3.0_1655922278586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_long","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_long","ko") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.electra").predict("""내 이름은 무엇입니까?|||"제 이름은 클라라이고 저는 버클리에 살고 있습니다.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_long| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|419.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sehandev/koelectra-long-qa --- layout: model title: Legal Whereas Clause Binary Classifier (md) author: John Snow Labs name: legclf_whereas_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `whereas` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `whereas` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_whereas_md_en_1.0.0_3.0_1669376529983.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_whereas_md_en_1.0.0_3.0_1669376529983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_whereas_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[whereas]| |[other]| |[other]| |[whereas]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_whereas_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support other 0.93 1.00 0.96 39 whereas 1.00 0.92 0.96 38 accuracy 0.96 77 macro avg 0.96 0.96 0.96 77 weighted avg 0.96 0.96 0.96 77 ``` --- layout: model title: Translate English to Semitic languages Pipeline author: John Snow Labs name: translate_en_sem date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, sem, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `sem` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_sem_xx_2.7.0_2.4_1609690194050.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_sem_xx_2.7.0_2.4_1609690194050.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_sem", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_sem", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.sem').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_sem| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Clinical Entities (ner_jsl_greedy_biobert) author: John Snow Labs name: ner_jsl_greedy_biobert_pipeline date: 2023-03-20 tags: [ner, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_greedy_biobert](https://nlp.johnsnowlabs.com/2021/08/13/ner_jsl_greedy_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_pipeline_en_4.3.0_3.2_1679310105776.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_pipeline_en_4.3.0_3.2_1679310105776.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_jsl_greedy_biobert_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_jsl_greedy_biobert_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biobert_jsl_greedy.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 1 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.9488 | | 2 | male | 38 | 41 | Gender | 0.9978 | | 3 | for 2 days | 48 | 57 | Duration | 0.7709 | | 4 | congestion | 62 | 71 | Symptom | 0.5467 | | 5 | mom | 75 | 77 | Gender | 0.9355 | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | 0.327867 | | 7 | nares | 135 | 139 | External_body_part_or_region | 0.8963 | | 8 | she | 147 | 149 | Gender | 0.995 | | 9 | mild problems with his breathing while feeding | 168 | 213 | Symptom | 0.588714 | | 10 | perioral cyanosis | 237 | 253 | Symptom | 0.58635 | | 11 | retractions | 258 | 268 | Symptom | 0.9864 | | 12 | One day ago | 272 | 282 | RelativeDate | 0.755833 | | 13 | mom | 285 | 287 | Gender | 0.9956 | | 14 | tactile temperature | 304 | 322 | Symptom | 0.10505 | | 15 | Tylenol | 345 | 351 | Drug | 0.9496 | | 16 | Baby | 354 | 357 | Age | 0.976 | | 17 | decreased p.o. intake | 377 | 397 | Symptom | 0.448125 | | 18 | His | 400 | 402 | Gender | 0.999 | | 19 | q.2h. to 5 to 10 minutes | 450 | 473 | Frequency | 0.298843 | | 20 | his | 488 | 490 | Gender | 0.9976 | | 21 | respiratory congestion | 492 | 513 | VS_Finding | 0.6158 | | 22 | He | 516 | 517 | Gender | 0.9998 | | 23 | tired | 550 | 554 | Symptom | 0.8912 | | 24 | fussy | 569 | 573 | Symptom | 0.9541 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_greedy_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm language: en nav_key: models repository: clinical/models date: 2020-11-27 task: Entity Resolution edition: Healthcare NLP 2.6.4 spark_version: 2.4 tags: [clinical,entity_resolution,en] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model maps extracted medical entities to RxNorm codes using chunk embeddings. {:.h2_title} ## Predicted Entities RxNorm Codes and their normalized definition with ``sbiobert_base_cased_mli`` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_2.6.4_2.4_1606235763316.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_2.6.4_2.4_1606235763316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, rxnorm_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala ... val chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, rxnorm_resolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 386165| 0.1567|hypercal:::hypersed:::hypertears:::hyperstat...|386165:::217667::...| |chronic renal ins...| 83|109| PROBLEM| 218689| 0.1036|nephro calci:::dialysis solutions:::creatini...|218689:::3310:::2...| | COPD| 113|116| PROBLEM|1539999| 0.1644|broncomar dm:::acne medication:::carbon mono...|1539999:::214981:...| | gastritis| 120|128| PROBLEM| 225965| 0.1983|gastroflux:::gastroflux oral product:::uceri...|225965:::1176661:...| | TIA| 136|138| PROBLEM|1089812| 0.0625|thera tears:::thiotepa injection:::nature's ...|1089812:::1660003...| |a non-ST elevatio...| 182|202| PROBLEM| 218767| 0.1007|non-aspirin pm:::aspirin-free:::non aspirin ...|218767:::215440::...| |Guaiac positive s...| 208|229| PROBLEM|1294361| 0.0820|anusol rectal product:::anusol hc rectal pro...|1294361:::1166715...| |cardiac catheteri...| 295|317| TEST| 385247| 0.1566|cardiacap:::cardiology pack:::cardizem:::car...|385247:::545063::...| | PTCA| 324|327|TREATMENT| 8410| 0.0867|alteplase:::reteplase:::pancuronium:::tripe ...|8410:::76895:::78...| | mid LAD lesion| 332|345| PROBLEM| 151672| 0.0549|dulcolax:::lazerformalyde:::linaclotide:::du...|151672:::217985::...| +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_rxnorm | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.4 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on November 2020 RxNorm Clinical Drugs ontology graph with ``sbiobert_base_cased_mli`` embeddings. https://www.nlm.nih.gov/pubs/techbull/nd20/brief/nd20_rx_norm_november_release.html --- layout: model title: English asr_wav2vec2_cetuc_sid_voxforge_mls_1 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: asr_wav2vec2_cetuc_sid_voxforge_mls_1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_cetuc_sid_voxforge_mls_1` is a English model originally trained by joaoalvarenga. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_cetuc_sid_voxforge_mls_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_cetuc_sid_voxforge_mls_1_en_4.2.0_3.0_1664023215604.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_cetuc_sid_voxforge_mls_1_en_4.2.0_3.0_1664023215604.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_cetuc_sid_voxforge_mls_1", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_cetuc_sid_voxforge_mls_1", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_cetuc_sid_voxforge_mls_1| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Multilingual BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_ar_hi_dev date: 2022-07-07 tags: [xx, open_source, bert, question_answering] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-ar-hi` is a Multilingual model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_ar_hi_dev_xx_4.0.0_3.0_1657189808354.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_ar_hi_dev_xx_4.0.0_3.0_1657189808354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_ar_hi_dev","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_ar_hi_dev","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_ar_hi_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-ar-hi --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from nlp-en-es) author: John Snow Labs name: roberta_qa_nlp_en_es_base_bne_finetuned_s_c date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-finetuned-sqac` is a Spanish model originally trained by `nlp-en-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_nlp_en_es_base_bne_finetuned_s_c_es_4.2.4_3.0_1669985738926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_nlp_en_es_base_bne_finetuned_s_c_es_4.2.4_3.0_1669985738926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_nlp_en_es_base_bne_finetuned_s_c","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_nlp_en_es_base_bne_finetuned_s_c","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_nlp_en_es_base_bne_finetuned_s_c| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nlp-en-es/roberta-base-bne-finetuned-sqac --- layout: model title: Irish Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:34:00 +0800 task: Lemmatization language: ga edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, ga] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_ga_2.5.5_2.4_1596054397576.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_ga_2.5.5_2.4_1596054397576.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "ga") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Seachas a bheith ina rí ar an tuaisceart, is dochtúir Sasanach é John Snow agus ceannaire i bhforbairt ainéistéise agus sláinteachas míochaine.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "ga") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Seachas a bheith ina rí ar an tuaisceart, is dochtúir Sasanach é John Snow agus ceannaire i bhforbairt ainéistéise agus sláinteachas míochaine.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Seachas a bheith ina rí ar an tuaisceart, is dochtúir Sasanach é John Snow agus ceannaire i bhforbairt ainéistéise agus sláinteachas míochaine."""] lemma_df = nlu.load('ga.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=6, result='Seachas', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=8, end=8, result='a', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=10, end=15, result='bheith', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=17, end=19, result='i', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=21, end=22, result='rí', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|ga| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English DistilBertForQuestionAnswering model (from hcy11) author: John Snow Labs name: distilbert_qa_hcy11_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `hcy11`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_hcy11_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725400512.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_hcy11_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725400512.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hcy11_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hcy11_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_hcy11").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hcy11_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hcy11/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC5CDR_Chem_Modified_BioBERT_512 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC5CDR-Chem-Modified-BioBERT-512` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CDR_Chem_Modified_BioBERT_512_en_4.0.0_3.0_1657109320849.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CDR_Chem_Modified_BioBERT_512_en_4.0.0_3.0_1657109320849.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC5CDR_Chem_Modified_BioBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC5CDR-Chem-Modified-BioBERT-512 --- layout: model title: Legal Asset Purchase Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_asset_purchase_agreement date: 2022-11-10 tags: [en, legal, classification, licensed, agreement] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_asset_purchase_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `asset-purchase-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `asset-purchase-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_asset_purchase_agreement_en_1.0.0_3.0_1668104092936.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_asset_purchase_agreement_en_1.0.0_3.0_1668104092936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[asset-purchase-agreement]| |[other]| |[other]| |[asset-purchase-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_asset_purchase_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support asset-purchase-agreement 0.96 0.96 0.96 27 other 0.99 0.99 0.99 85 accuracy - - 0.98 112 macro-avg 0.98 0.98 0.98 112 weighted-avg 0.98 0.98 0.98 112 ``` --- layout: model title: Finnish XLMRobertaForTokenClassification Base Cased model (from tner) author: John Snow Labs name: xlmroberta_ner_base_fin date: 2022-08-13 tags: [fi, open_source, xlm_roberta, ner] task: Named Entity Recognition language: fi edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-fin` is a Finnish model originally trained by `tner`. ## Predicted Entities `other`, `person`, `location`, `organization` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_fin_fi_4.1.0_3.0_1660426752654.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_fin_fi_4.1.0_3.0_1660426752654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_fin","fi") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_fin","fi") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_fin| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fi| |Size:|773.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/tner/xlm-roberta-base-fin - https://github.com/asahi417/tner --- layout: model title: Ukrainian T5ForConditionalGeneration Cased model (from ukr-models) author: John Snow Labs name: t5_uk_summarizer date: 2023-01-31 tags: [uk, open_source, t5, tensorflow] task: Text Generation language: uk edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `uk-summarizer` is a Ukrainian model originally trained by `ukr-models`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_uk_summarizer_uk_4.3.0_3.0_1675157739525.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_uk_summarizer_uk_4.3.0_3.0_1675157739525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_uk_summarizer","uk") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_uk_summarizer","uk") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_uk_summarizer| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|uk| |Size:|995.5 MB| ## References - https://huggingface.co/ukr-models/uk-summarizer --- layout: model title: Translate English to Chinese Pipeline author: John Snow Labs name: translate_en_zh date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, zh, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `zh` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_zh_xx_2.7.0_2.4_1609686009785.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_zh_xx_2.7.0_2.4_1609686009785.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_zh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_zh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.zh').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_zh| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Appointment Clause Binary Classifier author: John Snow Labs name: legclf_appointment_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `appointment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `appointment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_appointment_clause_en_1.0.0_3.2_1660122120413.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_appointment_clause_en_1.0.0_3.2_1660122120413.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_appointment_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[appointment]| |[other]| |[other]| |[appointment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_appointment_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support appointment 0.86 0.94 0.90 32 other 0.98 0.95 0.96 101 accuracy - - 0.95 133 macro-avg 0.92 0.94 0.93 133 weighted-avg 0.95 0.95 0.95 133 ``` --- layout: model title: Japanese Bert Embeddings (Base, Character Tokenization, Whole Word Masking) author: John Snow Labs name: bert_embeddings_bert_base_japanese_char_whole_word_masking date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-char-whole-word-masking` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_whole_word_masking_ja_3.4.2_3.0_1649674360241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_whole_word_masking_ja_3.4.2_3.0_1649674360241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_char_whole_word_masking| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|334.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking - https://github.com/google-research/bert - https://github.com/cl-tohoku/bert-japanese/tree/v1.0 - https://github.com/attardi/wikiextractor - https://taku910.github.io/mecab/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: Fast Neural Machine Translation Model from English to Tonga (Zambezi) author: John Snow Labs name: opus_mt_en_toi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, toi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `toi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_toi_xx_2.7.0_2.4_1609166942717.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_toi_xx_2.7.0_2.4_1609166942717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_toi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Warranty Clause Binary Classifier (CUAD dataset) author: John Snow Labs name: legclf_cuad_warranty_clause date: 2022-10-18 tags: [warranty, clause, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `warranty` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. There are other models in this dataset with similar title, but the difference is the dataset it was trained on. This one was trained with `cuad` dataset. ## Predicted Entities `warranty`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_warranty_clause_en_1.0.0_3.0_1666097671097.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_warranty_clause_en_1.0.0_3.0_1666097671097.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cuad_warranty_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["Sutro represents and warrants that:\n\n7.3.1..."]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[warranty]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_warranty_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References CUAD dataset ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.96 0.96 27 warranty 0.93 0.93 0.93 14 accuracy - - 0.95 41 macro-avg 0.95 0.95 0.95 41 weighted-avg 0.95 0.95 0.95 41 ``` --- layout: model title: Pipeline to Detect PHI for Deidentification author: John Snow Labs name: ner_deid_augmented_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_augmented](https://nlp.johnsnowlabs.com/2021/03/31/ner_deid_augmented_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_augmented_pipeline_en_3.4.1_3.0_1647864550318.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_augmented_pipeline_en_3.4.1_3.0_1647864550318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_augmented_pipeline", "en", "clinical/models") pipeline.annotate("HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_augmented_pipeline", "en", "clinical/models") pipeline.annotate("HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.ner_augmented.pipeline").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad2_boolq_baseline","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad2_boolq_baseline","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_squad2_boolq_baseline| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/shahrukhx01/roberta-base-squad2-boolq-baseline --- layout: model title: Translate Seychellois Creole to English Pipeline author: John Snow Labs name: translate_crs_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, crs, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `crs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_crs_en_xx_2.7.0_2.4_1609690310342.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_crs_en_xx_2.7.0_2.4_1609690310342.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_crs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_crs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.crs.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_crs_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_cline_emanuals_tech date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cline-emanuals-techqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_emanuals_tech_en_4.3.0_3.0_1674209326690.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_emanuals_tech_en_4.3.0_3.0_1674209326690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cline_emanuals_tech","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cline_emanuals_tech","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cline_emanuals_tech| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/cline-emanuals-techqa --- layout: model title: Vietnamese DistilBERT Base Cased Embeddings author: John Snow Labs name: distilbert_base_cased date: 2022-01-13 tags: [embeddings, distilbert, vietnamese, vi, open_source] task: Embeddings language: vi edition: Spark NLP 3.3.4 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This embeddings model was imported from `Hugging Face`. It's a custom version of `distilbert_base_multilingual_cased` and it gives the same representations produced by the original model which preserves the original accuracy. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_cased_vi_3.3.4_3.0_1642064850307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_cased_vi_3.3.4_3.0_1642064850307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... distilbert = DistilBertEmbeddings.pretrained("distilbert_base_cased", "vi")\ .setInputCols(["sentence",'token'])\ .setOutputCol("embeddings") nlpPipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, distilbert]) text = """Tôi yêu Spark NLP""" data = spark.createDataFrame([[text]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala ... val embeddings = DistilBertEmbeddings.pretrained("distilbert_base_cased", "vi") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("Tôi yêu Spark NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("vi.embed.distilbert.cased").predict("""Tôi yêu Spark NLP""") ```

## Results ```bash +-----+--------------------+ |token| embeddings| +-----+--------------------+ | Tôi|[-0.38760236, -0....| | yêu|[-0.3357051, -0.5...| |Spark|[-0.20642707, -0....| | NLP|[-0.013280544, -0...| +-----+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_cased| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|vi| |Size:|211.6 MB| |Case sensitive:|false| --- layout: model title: French CamemBert Embeddings (from Sonny) author: John Snow Labs name: camembert_embeddings_Sonny_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Sonny`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Sonny_generic_model_fr_3.4.4_3.0_1653986957057.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Sonny_generic_model_fr_3.4.4_3.0_1653986957057.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Sonny_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Sonny/dummy-model --- layout: model title: French CamemBert Embeddings (from elusive-magnolia) author: John Snow Labs name: camembert_embeddings_elusive_magnolia_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `elusive-magnolia`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_elusive_magnolia_generic_model_fr_3.4.4_3.0_1653988370340.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_elusive_magnolia_generic_model_fr_3.4.4_3.0_1653988370340.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_elusive_magnolia_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/elusive-magnolia/dummy-model --- layout: model title: Translate English to Tuvaluan Pipeline author: John Snow Labs name: translate_en_tvl date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tvl, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tvl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tvl_xx_2.7.0_2.4_1609686360411.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tvl_xx_2.7.0_2.4_1609686360411.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tvl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tvl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tvl').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tvl| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Basque to English Pipeline author: John Snow Labs name: translate_eu_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, eu, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `eu` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_eu_en_xx_2.7.0_2.4_1609686527199.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_eu_en_xx_2.7.0_2.4_1609686527199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_eu_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_eu_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.eu.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_eu_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from AlirezaBaneshi) author: John Snow Labs name: roberta_qa_autotrain_test2_756523213 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-test2-756523213` is a English model originally trained by `AlirezaBaneshi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523213_en_4.0.0_3.0_1655727630639.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523213_en_4.0.0_3.0_1655727630639.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523213","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_autotrain_test2_756523213","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.756523213.by_AlirezaBaneshi").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_autotrain_test2_756523213| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|415.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AlirezaBaneshi/autotrain-test2-756523213 --- layout: model title: Fast Neural Machine Translation Model from English to French author: John Snow Labs name: opus_mt_en_fr date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, fr, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `fr` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_fr_xx_2.7.0_2.4_1609166836357.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_fr_xx_2.7.0_2.4_1609166836357.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_fr| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Living Species (w2v_cc_300d) author: John Snow Labs name: ner_living_species_pipeline date: 2023-03-13 tags: [gl, ner, clinical, licensed] task: Named Entity Recognition language: gl edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species](https://nlp.johnsnowlabs.com/2022/06/23/ner_living_species_gl_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_gl_4.3.0_3.2_1678704830024.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_gl_4.3.0_3.2_1678704830024.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_pipeline", "gl", "clinical/models") text = '''Muller de 45 anos, sen antecedentes médicos de interese, que foi remitida á consulta de dermatoloxía de urxencias por lesións faciales de tres semanas de evolución. A paciente non presentaba lesións noutras localizaciones nin outra clínica de interese. No seu centro de saúde prescribíronlle corticoides tópicos ante a sospeita de picaduras de artrópodos e unha semana despois, antivirales orais baixo o diagnóstico de posible infección herpética. As lesións interferían de forma notable na súa vida persoal e profesional xa que traballaba de face ao púbico. Unha semana máis tarde o diagnóstico foi confirmado ao resultar o cultivo positivo a Staphylococcus aureus.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_pipeline", "gl", "clinical/models") val text = "Muller de 45 anos, sen antecedentes médicos de interese, que foi remitida á consulta de dermatoloxía de urxencias por lesións faciales de tres semanas de evolución. A paciente non presentaba lesións noutras localizaciones nin outra clínica de interese. No seu centro de saúde prescribíronlle corticoides tópicos ante a sospeita de picaduras de artrópodos e unha semana despois, antivirales orais baixo o diagnóstico de posible infección herpética. As lesións interferían de forma notable na súa vida persoal e profesional xa que traballaba de face ao púbico. Unha semana máis tarde o diagnóstico foi confirmado ao resultar o cultivo positivo a Staphylococcus aureus." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:------------|-------------:| | 0 | Muller | 0 | 5 | HUMAN | 0.9998 | | 1 | paciente | 167 | 174 | HUMAN | 0.9985 | | 2 | artrópodos | 344 | 353 | SPECIES | 0.9647 | | 3 | antivirales | 378 | 388 | SPECIES | 0.8854 | | 4 | herpética | 437 | 445 | SPECIES | 0.9592 | | 5 | púbico | 551 | 556 | HUMAN | 0.7293 | | 6 | Staphylococcus aureus | 644 | 664 | SPECIES | 0.87005 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|gl| |Size:|794.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Hindi BertForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: bert_embeddings_indic_transformers date: 2022-12-02 tags: [hi, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: hi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-hi-bert` is a Hindi model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_hi_4.2.4_3.0_1670022367639.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_hi_4.2.4_3.0_1670022367639.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","hi") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","hi") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_uk_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|uk| |Size:|195.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-uk-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Lemmatizer (Luxembourgish, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, lb] task: Lemmatization language: lb edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Luxembourgish Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_lb_3.4.1_3.0_1646316561258.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_lb_3.4.1_3.0_1646316561258.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------+ |result | +--------------------------------------+ |[dir, sidd, net, besseren, wéien, ech]| +--------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|lb| |Size:|3.9 MB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becasv2_3 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becasv2-3` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_3_en_4.3.0_3.0_1672767723393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_3_en_4.3.0_3.0_1672767723393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becasv2_3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becasv2-3 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from bdickson) author: John Snow Labs name: distilbert_qa_bdickson_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `bdickson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_bdickson_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770181883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_bdickson_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770181883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bdickson_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bdickson_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_bdickson_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/bdickson/distilbert-base-uncased-finetuned-squad --- layout: model title: German Bert Embeddings(Cased) author: John Snow Labs name: bert_embeddings_bert_base_german_dbmdz_cased date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-german-dbmdz-cased` is a German model orginally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_dbmdz_cased_de_3.4.2_3.0_1649676089568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_dbmdz_cased_de_3.4.2_3.0_1649676089568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_german_dbmdz_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/bert-base-german-dbmdz-cased --- layout: model title: Translate Estonian to English Pipeline author: John Snow Labs name: translate_et_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, et, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `et` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_et_en_xx_2.7.0_2.4_1609699041974.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_et_en_xx_2.7.0_2.4_1609699041974.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_et_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_et_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.et.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_et_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav TFWav2Vec2ForCTC from vai6hav author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav` is a English model originally trained by vai6hav. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_en_4.2.0_3.0_1664113051189.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_en_4.2.0_3.0_1664113051189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-8` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8_en_4.0.0_3.0_1654191650975.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8_en_4.0.0_3.0_1654191650975.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_512d_seed_8").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|387.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-8 --- layout: model title: Sentence Entity Resolver for Snomed Concepts, Body Structure Version (sbiobert_base_cased_mli) author: John Snow Labs name: sbiobertresolve_snomed_bodyStructure date: 2021-06-15 tags: [snomed, en, clinical, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical (anatomical structures) entities to Snomed codes (body structure version) using sentence embeddings. ## Predicted Entities Snomed Codes and their normalized definition with `sbiobert_base_cased_mli` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_bodyStructure_en_3.1.0_3.0_1623774132614.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_bodyStructure_en_3.1.0_3.0_1623774132614.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_snomed_bodyStructure``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Disease_Syndrome_Disorder, External_body_part_or_region``` set in ```.setWhiteList()```. ```sbiobertresolve_snomed_bodyStructure``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. No need to set ```.setWhiteList()```. Merge ner_jsl and ner_anatomy_coarse model chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") jsl_sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli','en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_snomed_bodyStructure, "en", "clinical/models) \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("snomed_code") snomed_pipelineModel = PipelineModel( stages = [ documentAssembler, jsl_sbert_embedder, snomed_resolver]) snomed_lp = LightPipeline(snomed_pipelineModel) result = snomed_lp.fullAnnotate("Amputation stump") ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_bodyStructure", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("snomed_code") val snomed_pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, snomed_resolver)) val snomed_lp = LightPipeline(snomed_pipelineModel) val result = snomed_lp.fullAnnotate("Amputation stump") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed_body_structure").predict("""sbiobertresolve_snomed_bodyStructure, """) ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:-----------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------| | 0 | amputation stump | 38033009 | [Amputation stump, Amputation stump of upper limb, Amputation stump of left upper limb, Amputation stump of lower limb, Amputation stump of left lower limb, Amputation stump of right upper limb, Amputation stump of right lower limb, ...]| ['38033009', '771359009', '771364008', '771358001', '771367001', '771365009', '771368006', ...] | ['0.0000', '0.0773', '0.0858', '0.0863', '0.0905', '0.0911', '0.0972', ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_bodyStructure| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[snomed_code]| |Language:|en| |Case sensitive:|true| ## Data Source https://www.snomed.org/ --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_ner_778023879 date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-ner-778023879` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1677881870073.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1677881870073.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_ner_778023879| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-ner-778023879 --- layout: model title: Pipeline to Extract Negation and Uncertainty Entities from Spanish Medical Texts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_negation_uncertainty_pipeline date: 2023-03-20 tags: [es, clinical, licensed, token_classification, bert, ner, negation, uncertainty, linguistics] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_negation_uncertainty](https://nlp.johnsnowlabs.com/2022/08/11/bert_token_classifier_negation_uncertainty_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_negation_uncertainty_pipeline_es_4.3.0_3.2_1679298806721.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_negation_uncertainty_pipeline_es_4.3.0_3.2_1679298806721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_negation_uncertainty_pipeline", "es", "clinical/models") text = '''Con diagnóstico probable de cirrosis hepática (no conocida previamente) y peritonitis espontanea primaria con tratamiento durante 8 dias con ceftriaxona en el primer ingreso (no se realizó paracentesis control por escasez de liquido). Lesión tumoral en hélix izquierdo de 0,5 cms. de diámetro susceptible de ca basocelular perlado.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_negation_uncertainty_pipeline", "es", "clinical/models") val text = "Con diagnóstico probable de cirrosis hepática (no conocida previamente) y peritonitis espontanea primaria con tratamiento durante 8 dias con ceftriaxona en el primer ingreso (no se realizó paracentesis control por escasez de liquido). Lesión tumoral en hélix izquierdo de 0,5 cms. de diámetro susceptible de ca basocelular perlado." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------------------------------------------------|--------:|------:|:------------|-------------:| | 0 | probable | 16 | 23 | UNC | 0.999994 | | 1 | de cirrosis hepática | 25 | 44 | USCO | 0.999988 | | 2 | no | 47 | 48 | NEG | 0.999995 | | 3 | conocida previamente | 50 | 69 | NSCO | 0.999992 | | 4 | no | 175 | 176 | NEG | 0.999995 | | 5 | se realizó paracentesis control por escasez de liquido | 178 | 231 | NSCO | 0.999995 | | 6 | susceptible de | 293 | 306 | UNC | 0.999986 | | 7 | ca basocelular perlado | 308 | 329 | USCO | 0.99999 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_negation_uncertainty_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|410.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_3 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-3` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_3_en_4.3.0_3.0_1672767491512.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_3_en_4.3.0_3.0_1672767491512.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-3 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from k3nneth) author: John Snow Labs name: xlmroberta_ner_k3nneth_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `k3nneth`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_k3nneth_base_finetuned_panx_de_4.1.0_3.0_1660434928587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_k3nneth_base_finetuned_panx_de_4.1.0_3.0_1660434928587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_k3nneth_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_k3nneth_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_k3nneth_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/k3nneth/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal The closing Clause Binary Classifier author: John Snow Labs name: legclf_the_closing_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `the-closing` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `the-closing` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_the_closing_clause_en_1.0.0_3.2_1660123095618.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_the_closing_clause_en_1.0.0_3.2_1660123095618.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_the_closing_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[the-closing]| |[other]| |[other]| |[the-closing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_the_closing_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 105 the-closing 1.00 1.00 1.00 35 accuracy - - 1.00 140 macro-avg 1.00 1.00 1.00 140 weighted-avg 1.00 1.00 1.00 140 ``` --- layout: model title: Clinical Portuguese Bert Embeddiongs (Clinical) author: John Snow Labs name: biobert_embeddings_clinical date: 2022-04-11 tags: [biobert, embeddings, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BioBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `biobertpt-clin` is a Portuguese model orginally trained by `pucpr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_embeddings_clinical_pt_3.4.2_3.0_1649686994929.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_embeddings_clinical_pt_3.4.2_3.0_1649686994929.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_embeddings_clinical| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|667.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/pucpr/biobertpt-clin - https://aclanthology.org/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-42` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42_en_4.0.0_3.0_1654191455684.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42_en_4.0.0_3.0_1654191455684.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_seed_42").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_42| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|380.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-42 --- layout: model title: English asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu TFWav2Vec2ForCTC from adelgalu author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu` is a English model originally trained by adelgalu. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu_en_4.2.0_3.0_1664098872120.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu_en_4.2.0_3.0_1664098872120.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_adelgalu| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.3 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from maroo93) author: John Snow Labs name: bert_qa_squad1.1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad1.1` is a English model orginally trained by `maroo93`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad1.1_en_4.0.0_3.0_1654192132045.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad1.1_en_4.0.0_3.0_1654192132045.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad1.1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_maroo93").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad1.1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/maroo93/squad1.1 --- layout: model title: Pretrained Pipeline for Few-NERD-General NER Model author: John Snow Labs name: nerdl_fewnerd_100d_pipeline date: 2022-06-28 tags: [fewnerd, nerdl, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on Few-NERD model and it detects : `PERSON`, `ORGANIZATION`, `LOCATION`, `ART`, `BUILDING`, `PRODUCT`, `EVENT`, `OTHER` ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_pipeline_en_4.0.0_3.0_1656388980361.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_pipeline_en_4.0.0_3.0_1656388980361.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python fewnerd_pipeline = PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") fewnerd_pipeline.annotate("""The Double Down is a sandwich offered by Kentucky Fried Chicken restaurants. He did not see active service again until 1882, when he took part in the Anglo-Egyptian War, and was present at the battle of Tell El Kebir (September 1882), for which he was mentioned in dispatches, received the Egypt Medal with clasp and the 3rd class of the Order of Medjidie, and was appointed a Companion of the Order of the Bath (CB).""") ``` ```scala val pipeline = new PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") val result = pipeline.fullAnnotate("The Double Down is a sandwich offered by Kentucky Fried Chicken restaurants. He did not see active service again until 1882, when he took part in the Anglo-Egyptian War, and was present at the battle of Tell El Kebir (September 1882), for which he was mentioned in dispatches, received the Egypt Medal with clasp and the 3rd class of the Order of Medjidie, and was appointed a Companion of the Order of the Bath (CB).")(0) ```

## Results ```bash +-----------------------+------------+ |chunk |ner_label | +-----------------------+------------+ |Kentucky Fried Chicken |ORGANIZATION| |Anglo-Egyptian War |EVENT | |battle of Tell El Kebir|EVENT | |Egypt Medal |OTHER | |Order of Medjidie |OTHER | +-----------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_fewnerd_100d_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|167.3 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter - Finisher --- layout: model title: English RobertaForQuestionAnswering Cased model (from Ching) author: John Snow Labs name: roberta_qa_negation_detector date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `negation_detector` is a English model originally trained by `Ching`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_negation_detector_en_4.3.0_3.0_1674211601485.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_negation_detector_en_4.3.0_3.0_1674211601485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_negation_detector","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_negation_detector","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_negation_detector| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Ching/negation_detector --- layout: model title: Extract test entities (Voice of the Patients) author: John Snow Labs name: ner_vop_test_wip date: 2023-04-20 tags: [licensed, clinical, en, ner, vop, patient, test] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts test mentions from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Measurements`, `TestResult`, `Test`, `VitalTest` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_wip_en_4.4.0_3.0_1682013044617.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_wip_en_4.4.0_3.0_1682013044617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------|:------------| | thyroid levels | Test | | blood test | Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_test_wip| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.9 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I”m 20 year old girl. I”m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I”m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I”m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I”m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Measurements 100 58 30 130 0.63 0.77 0.69 TestResult 452 124 182 634 0.78 0.71 0.75 Test 1194 98 207 1401 0.92 0.85 0.89 VitalTest 195 20 23 218 0.91 0.89 0.90 macro_avg 1941 300 442 2383 0.81 0.80 0.81 micro_avg 1941 300 442 2383 0.87 0.81 0.84 ``` --- layout: model title: Sentiment Analysis of IMDB Reviews Pipeline (analyze_sentimentdl_use_imdb) author: John Snow Labs name: analyze_sentimentdl_use_imdb date: 2021-01-15 task: [Embeddings, Sentiment Analysis, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [en, pipeline, sentiment] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline to classify IMDB reviews in `neg` and `pos` classes using `tfhub_use` embeddings. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_use_imdb_en_2.7.1_2.4_1610723836151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_use_imdb_en_2.7.1_2.4_1610723836151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("analyze_sentimentdl_use_imdb", lang = "en") result = pipeline.fullAnnotate("Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("analyze_sentimentdl_use_imdb", lang = "en") val result = pipeline.fullAnnotate("Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!") ``` {:.nlu-block} ```python import nlu text = ["""Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!"""] sentiment_df = nlu.load('en.sentiment.imdb.use').predict(text, output_level='sentence') sentiment_df ```

## Results ```bash | | document | sentiment | |---:|---------------------------------------------------------------------------------------------------------:|--------------:| | | Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the | | | 0 | film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music | positive | | | was rad! Horror and sword fight freaks,buy this movie now! | | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|analyze_sentimentdl_use_imdb| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Language:|en| ## Included Models `tfhub_use`, `sentimentdl_use_imdb` --- layout: model title: Legal Deposit Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_deposit_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, deposit, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_deposit_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `deposit-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `deposit-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_deposit_agreement_bert_en_1.0.0_3.0_1670349380582.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_deposit_agreement_bert_en_1.0.0_3.0_1670349380582.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_deposit_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[deposit-agreement]| |[other]| |[other]| |[deposit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_deposit_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support deposit-agreement 0.97 0.97 0.97 36 other 0.98 0.98 0.98 65 accuracy - - 0.98 101 macro-avg 0.98 0.98 0.98 101 weighted-avg 0.98 0.98 0.98 101 ``` --- layout: model title: Legal Section headings Clause Binary Classifier author: John Snow Labs name: legclf_section_headings_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `section-headings` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `section-headings` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_section_headings_clause_en_1.0.0_3.2_1660122983672.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_section_headings_clause_en_1.0.0_3.2_1660122983672.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_section_headings_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[section-headings]| |[other]| |[other]| |[section-headings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_section_headings_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 159 section-headings 1.00 1.00 1.00 46 accuracy - - 1.00 205 macro-avg 1.00 1.00 1.00 205 weighted-avg 1.00 1.00 1.00 205 ``` --- layout: model title: Extract Anatomical Entities from Voice of the Patient Documents (embeddings_clinical_large) author: John Snow Labs name: ner_vop_anatomy_emb_clinical_large date: 2023-06-06 tags: [licensed, clinical, en, ner, vop, anatomy] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts anatomical terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `BodyPart`, `Laterality` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_emb_clinical_large_en_4.4.3_3.0_1686074062221.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_emb_clinical_large_en_4.4.3_3.0_1686074062221.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------|:------------| | muscle | BodyPart | | neck | BodyPart | | trapezius | BodyPart | | head | BodyPart | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_anatomy_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_large| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 BodyPart 2725 236 175 2900 0.92 0.94 0.93 Laterality 546 62 82 628 0.90 0.87 0.88 macro_avg 3271 298 257 3528 0.91 0.90 0.90 micro_avg 3271 298 257 3528 0.92 0.93 0.92 ``` --- layout: model title: Part of Speech for Bulgarian author: John Snow Labs name: pos_btb date: 2021-03-23 tags: [pos, bg, open_source] supported: true task: Part of Speech Tagging language: bg edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_btb_bg_2.7.5_2.4_1616506894131.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_btb_bg_2.7.5_2.4_1616506894131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_btb", "bg")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Столица на Република България е град София .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_btb", "bg") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("Столица на Република България е град София .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Столица на Република България е град София .""] token_df = nlu.load('bg.pos.btb').predict(text) token_df ```

## Results ```bash +--------------------------------------------+-------------------------------------------------+ |text |result | +--------------------------------------------+-------------------------------------------------+ |Столица на Република България е град София .|[NOUN, ADP, NOUN, PROPN, AUX, NOUN, PROPN, PUNCT]| +--------------------------------------------+-------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_btb| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|bg| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.89 | 0.87 | 0.88 | 1377 | | ADP | 0.95 | 0.95 | 0.95 | 2238 | | ADV | 0.94 | 0.92 | 0.93 | 671 | | AUX | 0.98 | 0.97 | 0.97 | 916 | | CCONJ | 0.96 | 0.95 | 0.96 | 467 | | DET | 0.91 | 0.88 | 0.90 | 273 | | INTJ | 1.00 | 1.00 | 1.00 | 1 | | NOUN | 0.92 | 0.93 | 0.93 | 3486 | | NUM | 0.89 | 0.87 | 0.88 | 223 | | PART | 0.98 | 0.96 | 0.97 | 210 | | PRON | 0.97 | 0.97 | 0.97 | 981 | | PROPN | 0.88 | 0.89 | 0.89 | 805 | | PUNCT | 0.95 | 0.96 | 0.95 | 2268 | | SCONJ | 0.98 | 0.97 | 0.98 | 156 | | VERB | 0.95 | 0.94 | 0.94 | 1652 | | accuracy | | | 0.94 | 15724 | | macro avg | 0.94 | 0.94 | 0.94 | 15724 | | weighted avg | 0.94 | 0.94 | 0.94 | 15724 | ``` --- layout: model title: English asr_wav2vec2_xlsr_53_phon TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_xlsr_53_phon date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_53_phon` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_53_phon_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_53_phon_en_4.2.0_3.0_1664109116417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_53_phon_en_4.2.0_3.0_1664109116417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_53_phon", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_53_phon", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_53_phon| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|756.9 MB| --- layout: model title: German XlmRoBertaForQuestionAnswering (from saattrupdan) author: John Snow Labs name: xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan date: 2022-06-24 tags: [de, open_source, question_answering, xlmroberta] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlmr-base-texas-squad-de` is a German model originally trained by `saattrupdan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan_de_4.0.0_3.0_1656062956033.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan_de_4.0.0_3.0_1656062956033.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan","de") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.squad_de_tuned.xlmr_roberta.base.by_saattrupdan").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlmr_base_texas_squad_de_de_saattrupdan| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|de| |Size:|874.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saattrupdan/xlmr-base-texas-squad-de --- layout: model title: Word2Vec Embeddings in Sardinian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, sc, open_source] task: Embeddings language: sc edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sc_3.4.1_3.0_1647455351489.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sc_3.4.1_3.0_1647455351489.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|sc| |Size:|74.3 MB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Legal Permitted Use Clause Binary Classifier author: John Snow Labs name: legclf_permitted_use_clause date: 2023-02-13 tags: [en, legal, classification, permitted, use, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `permitted_use` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `permitted_use`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_permitted_use_clause_en_1.0.0_3.0_1676305311848.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_permitted_use_clause_en_1.0.0_3.0_1676305311848.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_permitted_use_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[permitted_use]| |[other]| |[other]| |[permitted_use]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_permitted_use_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 5 permitted_use 1.00 1.00 1.00 11 accuracy - - 1.00 16 macro-avg 1.00 1.00 1.00 16 weighted-avg 1.00 1.00 1.00 16 ``` --- layout: model title: Legal Bonus Clause Binary Classifier author: John Snow Labs name: legclf_bonus_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `bonus` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `bonus` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_bonus_clause_en_1.0.0_3.2_1660122172503.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_bonus_clause_en_1.0.0_3.2_1660122172503.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_bonus_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[bonus]| |[other]| |[other]| |[bonus]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_bonus_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support bonus 0.97 0.95 0.96 38 other 0.98 0.99 0.98 95 accuracy - - 0.98 133 macro-avg 0.98 0.97 0.97 133 weighted-avg 0.98 0.98 0.98 133 ``` --- layout: model title: Detect Diagnoses and Procedures (Spanish) author: John Snow Labs name: ner_diag_proc date: 2021-03-31 tags: [ner, clinical, licensed, es] task: Named Entity Recognition language: es edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN.Pretrained named entity recognition deep learning model for diagnostics and procedures in spanish ## Predicted Entities ``DIAGNOSTICO``, ``PROCEDIMIENTO``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DIAG_PROC_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DIAG_PROC_ES.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diag_proc_es_3.0.0_3.0_1617208422892.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diag_proc_es_3.0.0_3.0_1617208422892.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embed = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") model = MedicalNerModel.pretrained("ner_diag_proc","es","clinical/models")\ .setInputCols("sentence","token","word_embeddings")\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embed, model, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embed = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") val model = MedicalNerModel.pretrained("ner_diag_proc","es","clinical/models") .setInputCols(Array("sentence","token","word_embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embed, model, ner_converter)) val data = Seq("""HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner").predict("""HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.""") ```

## Results ```bash +----------------------+-------------+ |chunk |ner_label | +----------------------+-------------+ |ENFERMEDAD |DIAGNOSTICO | |cáncer de vejiga |DIAGNOSTICO | |resección |PROCEDIMIENTO| |cistectomía |PROCEDIMIENTO| |estrés |DIAGNOSTICO | |infarto subendocárdico|DIAGNOSTICO | |enfermedad |DIAGNOSTICO | |arterias coronarias |DIAGNOSTICO | +----------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_diag_proc| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| ## Benchmarking ```bash +-------------+------+------+------+------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +-------------+------+------+------+------+---------+------+------+ |PROCEDIMIENTO|2299.0|1103.0| 860.0|3159.0| 0.6758|0.7278|0.7008| | DIAGNOSTICO|6623.0|1364.0|2974.0|9597.0| 0.8292|0.6901|0.7533| +-------------+------+------+------+------+---------+------+------+ +------------------+ | macro| +------------------+ |0.7270531284138397| +------------------+ +------------------+ | micro| +------------------+ |0.7402992400932049| +------------------+ ``` --- layout: model title: English BertForQuestionAnswering model (from motiondew) author: John Snow Labs name: bert_qa_bert_finetuned_lr2_e5_b16_ep2 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-lr2-e5-b16-ep2` is a English model orginally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_finetuned_lr2_e5_b16_ep2_en_4.0.0_3.0_1654535195058.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_finetuned_lr2_e5_b16_ep2_en_4.0.0_3.0_1654535195058.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_finetuned_lr2_e5_b16_ep2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_finetuned_lr2_e5_b16_ep2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_motiondew").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_finetuned_lr2_e5_b16_ep2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-finetuned-lr2-e5-b16-ep2 --- layout: model title: Spanish RobertaForQuestionAnswering (from mrm8488) author: John Snow Labs name: roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-finetuned-sqac` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac_es_4.0.0_3.0_1655729996691.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac_es_4.0.0_3.0_1655729996691.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.sqac.roberta.base.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_mrm8488_roberta_base_bne_finetuned_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|460.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/roberta-base-bne-finetuned-sqac - https://paperswithcode.com/sota?task=Question+Answering&dataset=sqac --- layout: model title: English RobertaForQuestionAnswering Cased model (from clementgyj) author: John Snow Labs name: roberta_qa_finetuned_squad_50k date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-squad-50k` is a English model originally trained by `clementgyj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_squad_50k_en_4.3.0_3.0_1674220438911.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_squad_50k_en_4.3.0_3.0_1674220438911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_squad_50k","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_squad_50k","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_squad_50k| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|462.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/clementgyj/roberta-finetuned-squad-50k --- layout: model title: English AlbertForQuestionAnswering model (from twmkn9) author: John Snow Labs name: albert_base_qa_squad2 date: 2022-06-15 tags: [open_source, albert, question_answering, en] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `albert-base-v2-squad2` is a English model originally trained by `twmkn9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_base_qa_squad2_en_4.0.0_3.0_1655294222450.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_base_qa_squad2_en_4.0.0_3.0_1655294222450.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_base_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_base_qa_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.span_question.albert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_base_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References https://huggingface.co/twmkn9/albert-base-v2-squad2 --- layout: model title: Clinical Deidentification author: John Snow Labs name: clinical_deidentification date: 2023-06-13 tags: [deidentification, pipeline, de, licensed, clinical] task: Pipeline Healthcare language: de edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from **German** medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `STREET`, `USERNAME`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE`, `CONTACT`, `ID`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `DLN`, `PLATE` entities. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_de_4.4.4_3.2_1686663693325.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_de_4.4.4_3.2_1686663693325.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "de", "clinical/models") sample = """Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300 """ result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = PretrainedPipeline("clinical_deidentification","de","clinical/models") val sample = "Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("de.deid.clinical").predict("""Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300 """) ```

## Results ```bash Results Masked with entity labels ------------------------------ Zusammenfassung : wird am Morgen des ins eingeliefert. Herr ist Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: Platte Kontonummer: SSN : Lizenznummer: Adresse : Masked with chars ------------------------------ Zusammenfassung : [************] wird am Morgen des [**************] ins [**********************] eingeliefert. Herr [************] ist ** Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: [*******] Platte [*****] Kontonummer: [********************] SSN : [**********] Lizenznummer: [*********] Adresse : [*****************] [***] Masked with fixed length chars ------------------------------ Zusammenfassung : **** wird am Morgen des **** ins **** eingeliefert. Herr **** ist **** Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: **** Platte **** Kontonummer: **** SSN : **** Lizenznummer: **** Adresse : **** **** Obfusceted ------------------------------ Zusammenfassung : Herrmann Kallert wird am Morgen des 11-26-1977 ins International Neuroscience eingeliefert. Herr Herrmann Kallert ist 79 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: 136704D357 Platte QA348G Kontonummer: 192837465738 SSN : 1310011981M454 Lizenznummer: XX123456 Adresse : Klingelhöferring 31206 {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|de| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: German asr_exp_w2v2t_vp_100k_s627 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2t_vp_100k_s627 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_vp_100k_s627` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2t_vp_100k_s627_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_vp_100k_s627_de_4.2.0_3.0_1664105815417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_vp_100k_s627_de_4.2.0_3.0_1664105815417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2t_vp_100k_s627", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2t_vp_100k_s627", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2t_vp_100k_s627| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739775556.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739775556.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_twostagetriplet_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|306.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0 --- layout: model title: Legal Rights And Freedoms Document Classifier (EURLEX) author: John Snow Labs name: legclf_rights_and_freedoms_bert date: 2023-03-06 tags: [en, legal, classification, clauses, rights_and_freedoms, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_rights_and_freedoms_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Rights_and_Freedoms or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Rights_and_Freedoms`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_rights_and_freedoms_bert_en_1.0.0_3.0_1678111839271.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_rights_and_freedoms_bert_en_1.0.0_3.0_1678111839271.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_rights_and_freedoms_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Rights_and_Freedoms]| |[Other]| |[Other]| |[Rights_and_Freedoms]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_rights_and_freedoms_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.89 0.85 0.87 39 Rights_and_Freedoms 0.79 0.85 0.81 26 accuracy - - 0.85 65 macro-avg 0.84 0.85 0.84 65 weighted-avg 0.85 0.85 0.85 65 ``` --- layout: model title: Portuguese asr_bp_lapsbm1_xlsr TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_bp_lapsbm1_xlsr date: 2022-09-26 tags: [wav2vec2, pt, audio, open_source, asr] task: Automatic Speech Recognition language: pt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bp_lapsbm1_xlsr` is a Portuguese model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bp_lapsbm1_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bp_lapsbm1_xlsr_pt_4.2.0_3.0_1664190605281.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bp_lapsbm1_xlsr_pt_4.2.0_3.0_1664190605281.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bp_lapsbm1_xlsr", "pt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bp_lapsbm1_xlsr", "pt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bp_lapsbm1_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|pt| |Size:|756.4 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Cushitic languages author: John Snow Labs name: opus_mt_en_cus date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, cus, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `cus` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_cus_xx_2.7.0_2.4_1609168898891.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_cus_xx_2.7.0_2.4_1609168898891.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_cus| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: BERT Sentence Embeddings (Large Uncased) author: John Snow Labs name: sent_bert_large_uncased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus. The details are described in the paper "[BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_large_uncased_en_2.6.0_2.4_1598347026632.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_large_uncased_en_2.6.0_2.4_1598347026632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_bert_large_uncased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_bert_large_uncased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.bert_large_uncased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_bert_large_uncased_embeddings I hate cancer [[-0.13290119171142578, -0.2996622622013092, -... Antibiotics aren't painkiller [[-0.13290119171142578, -0.2996622622013092, -... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_large_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/bert_uncased_L-24_H-1024_A-16/1](https://tfhub.dev/google/bert_uncased_L-24_H-1024_A-16/1) --- layout: model title: Chinese Part of Speech Tagger (from ckiplab) author: John Snow Labs name: bert_pos_bert_base_chinese_pos date: 2022-04-26 tags: [bert, pos, part_of_speech, zh, open_source] task: Part of Speech Tagging language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-chinese-pos` is a Chinese model orginally trained by `ckiplab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_chinese_pos_zh_3.4.2_3.0_1650993041893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_chinese_pos_zh_3.4.2_3.0_1650993041893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_chinese_pos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|zh| |Size:|381.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ckiplab/bert-base-chinese-pos - https://github.com/ckiplab/ckip-transformers - https://muyang.pro - https://ckip.iis.sinica.edu.tw - https://github.com/ckiplab/ckip-transformers - https://github.com/ckiplab/ckip-transformers --- layout: model title: Swedish Legal Roberta Embeddings author: John Snow Labs name: roberta_base_swedish_legal date: 2023-02-17 tags: [se, swedish, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: se edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-swedish-roberta-base` is a Swedish model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_swedish_legal_se_4.2.4_3.0_1676643288694.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_swedish_legal_se_4.2.4_3.0_1676643288694.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_swedish_legal", "se")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_swedish_legal", "se") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_swedish_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|se| |Size:|415.9 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-swedish-roberta-base --- layout: model title: Financial Financial conditions Item Binary Classifier author: John Snow Labs name: finclf_financial_conditions_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `financial_conditions` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `financial_conditions` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_financial_conditions_item_en_1.0.0_3.2_1660154420184.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_financial_conditions_item_en_1.0.0_3.2_1660154420184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_financial_conditions_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[financial_conditions]| |[other]| |[other]| |[financial_conditions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_financial_conditions_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support financial_conditions 0.83 0.73 0.78 245 other 0.75 0.84 0.80 237 accuracy - - 0.79 482 macro-avg 0.79 0.79 0.79 482 weighted-avg 0.79 0.79 0.79 482 ``` --- layout: model title: Legal Non Exclusivity Clause Binary Classifier author: John Snow Labs name: legclf_non_exclusivity_clause date: 2023-01-29 tags: [en, legal, classification, exclusivity, clauses, non_exclusivity, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `non-exclusivity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `non-exclusivity`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_non_exclusivity_clause_en_1.0.0_3.0_1675006033580.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_non_exclusivity_clause_en_1.0.0_3.0_1675006033580.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_non_exclusivity_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[non-exclusivity]| |[other]| |[other]| |[non-exclusivity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_non_exclusivity_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support non-exclusivity 0.93 0.96 0.95 27 other 0.97 0.95 0.96 39 accuracy - - 0.95 66 macro-avg 0.95 0.96 0.95 66 weighted-avg 0.96 0.95 0.95 66 ``` --- layout: model title: Finnish asr_wav2vec2_large_xlsr_finnish TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: asr_wav2vec2_large_xlsr_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_finnish` is a Finnish model originally trained by birgermoell. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_finnish_fi_4.2.0_3.0_1664021375004.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_finnish_fi_4.2.0_3.0_1664021375004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_finnish", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_finnish", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_finnish| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: Legal Intellectual property Clause Binary Classifier author: John Snow Labs name: legclf_intellectual_property_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `intellectual-property` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `intellectual-property` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_intellectual_property_clause_en_1.0.0_3.2_1660123623906.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_intellectual_property_clause_en_1.0.0_3.2_1660123623906.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_intellectual_property_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[intellectual-property]| |[other]| |[other]| |[intellectual-property]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_intellectual_property_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support intellectual-property 0.95 0.85 0.90 47 other 0.93 0.98 0.95 95 accuracy - - 0.94 142 macro-avg 0.94 0.92 0.93 142 weighted-avg 0.94 0.94 0.94 142 ``` --- layout: model title: German Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_large_german_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, de, open_source] task: Part of Speech Tagging language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-german-upos` is a German model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_german_upos_de_3.4.2_3.0_1652092375858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_german_upos_de_3.4.2_3.0_1652092375858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_large_german_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-large-german-upos - https://github.com/UniversalDependencies/UD_German-HDT - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-32-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8_en_4.0.0_3.0_1657185089636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8_en_4.0.0_3.0_1657185089636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-32-finetuned-squad-seed-8 --- layout: model title: Sentence Embeddings - sbert medium (tuned) author: John Snow Labs name: sbiobert_jsl_rxnorm_cased date: 2021-12-23 tags: [licensed, embeddings, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps sentences & documents to a 768 dimensional dense vector space by using average pooling on top of BioBert model. It's also fine-tuned on RxNorm dataset to help generalization over medication-related datasets. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_rxnorm_cased_en_3.3.4_2.4_1640271525048.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_rxnorm_cased_en_3.3.4_2.4_1640271525048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_jsl_rxnorm_cased", "en", "clinical/models")\ .setInputCols(["sentence"])\ .setOutputCol("sbioert_embeddings") ``` ```scala val sentence_embeddings = BertSentenceEmbeddings.pretrained('sbiobert_jsl_rxnorm_cased', 'en','clinical/models') .setInputCols("sentence") .setOutputCol("sbioert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.biobert.rxnorm").predict("""Put your text here.""") ```

## Results ```bash Gives a 768-dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobert_jsl_rxnorm_cased| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|402.0 MB| --- layout: model title: Pipeline to Detect Medication Entities, Assign Assertion Status and Find Relations author: John Snow Labs name: explain_clinical_doc_medication date: 2023-04-20 tags: [licensed, clinical, ner, en, assertion, relation_extraction, posology, medication] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline for detecting posology entities with the `ner_posology_large` NER model, assigning their assertion status with `assertion_jsl` model, and extracting relations between posology-related terminology with `posology_re` relation extraction model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_medication_en_4.3.0_3.2_1682017727303.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_medication_en_4.3.0_3.2_1682017727303.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("explain_clinical_doc_medication", "en", "clinical/models") text = '''The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2. She received a course of Bactrim for 14 days for UTI. She was prescribed 5000 units of Fragmin subcutaneously daily, and along with Lantus 40 units subcutaneously at bedtime.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("explain_clinical_doc_medication", "en", "clinical/models") val text = "The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2. She received a course of Bactrim for 14 days for UTI. She was prescribed 5000 units of Fragmin subcutaneously daily, and along with Lantus 40 units subcutaneously at bedtime." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_dco.clinical_medication.pipeline").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2. She received a course of Bactrim for 14 days for UTI. She was prescribed 5000 units of Fragmin subcutaneously daily, and along with Lantus 40 units subcutaneously at bedtime.""") ```

## Results ```bash +----+----------------+------------+ | | chunks | entities | |---:|:---------------|:-----------| | 0 | insulin | DRUG | | 1 | Bactrim | DRUG | | 2 | for 14 days | DURATION | | 3 | 5000 units | DOSAGE | | 4 | Fragmin | DRUG | | 5 | subcutaneously | ROUTE | | 6 | daily | FREQUENCY | | 7 | Lantus | DRUG | | 8 | 40 units | DOSAGE | | 9 | subcutaneously | ROUTE | | 10 | at bedtime | FREQUENCY | +----+----------------+------------+ +----+----------+------------+-------------+ | | chunks | entities | assertion | |---:|:---------|:-----------|:------------| | 0 | insulin | DRUG | Present | | 1 | Bactrim | DRUG | Past | | 2 | Fragmin | DRUG | Planned | | 3 | Lantus | DRUG | Planned | +----+----------+------------+-------------+ +----------------+-----------+------------+-----------+----------------+ | relation | entity1 | chunk1 | entity2 | chunk2 | |:---------------|:----------|:-----------|:----------|:---------------| | DRUG-DURATION | DRUG | Bactrim | DURATION | for 14 days | | DOSAGE-DRUG | DOSAGE | 5000 units | DRUG | Fragmin | | DRUG-ROUTE | DRUG | Fragmin | ROUTE | subcutaneously | | DRUG-FREQUENCY | DRUG | Fragmin | FREQUENCY | daily | | DRUG-DOSAGE | DRUG | Lantus | DOSAGE | 40 units | | DRUG-ROUTE | DRUG | Lantus | ROUTE | subcutaneously | | DRUG-FREQUENCY | DRUG | Lantus | FREQUENCY | at bedtime | +----------------+-----------+------------+-----------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_medication| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - NerConverterInternalModel - AssertionDLModel - PerceptronModel - DependencyParserModel - PosologyREModel --- layout: model title: Bemba (Zambia) asr_wav2vec2_large_xls_r_300m_bemba_fds TFWav2Vec2ForCTC from csikasote author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds date: 2022-09-24 tags: [wav2vec2, bem, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: bem edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_bemba_fds` is a Bemba (Zambia) model originally trained by csikasote. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds_bem_4.2.0_3.0_1664023955232.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds_bem_4.2.0_3.0_1664023955232.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds', lang = 'bem') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds", lang = "bem") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_bemba_fds| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|bem| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English image_classifier_vit_ViTFineTuned ViTForImageClassification from pthpth author: John Snow Labs name: image_classifier_vit_ViTFineTuned date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_ViTFineTuned` is a English model originally trained by pthpth. ## Predicted Entities `white_bread`, `brown_bread`, `cracker`, `aluminium_foil`, `linen`, `wool`, `corduroy`, `wood`, `lettuce_leaf`, `cotton`, `cork` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ViTFineTuned_en_4.1.0_3.0_1660167943982.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ViTFineTuned_en_4.1.0_3.0_1660167943982.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_ViTFineTuned", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_ViTFineTuned", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_ViTFineTuned| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Effectiveness Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_effectiveness_bert date: 2023-03-05 tags: [en, legal, classification, clauses, effectiveness, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Effectiveness` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Effectiveness`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_effectiveness_bert_en_1.0.0_3.0_1678050012884.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_effectiveness_bert_en_1.0.0_3.0_1678050012884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_effectiveness_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Effectiveness]| |[Other]| |[Other]| |[Effectiveness]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_effectiveness_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Effectiveness 0.88 0.92 0.90 24 Other 0.94 0.92 0.93 36 accuracy - - 0.92 60 macro-avg 0.91 0.92 0.91 60 weighted-avg 0.92 0.92 0.92 60 ``` --- layout: model title: German asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545 date: 2022-09-26 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545_de_4.2.0_3.0_1664191216048.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545_de_4.2.0_3.0_1664191216048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s545| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Part of Speech for Afrikaans author: John Snow Labs name: pos_afribooms date: 2021-03-16 tags: [af, open_source, pos] supported: true task: Part of Speech Tagging language: af edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_afribooms_af_2.7.5_2.4_1615903333785.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_afribooms_af_2.7.5_2.4_1615903333785.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_afribooms", "af")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_afribooms", "af") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer ,pos)) val data = Seq("Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .""] token_df = nlu.load('af.pos.afribooms').predict(text) token_df ```

## Results ```bash +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ |text |result | +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ |Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .|[DET, NOUN, PRON, VERB, AUX, PUNCT, AUX, ADJ, CCONJ, ADJ, ADP, NOUN, CCONJ, NOUN, AUX, PUNCT]| +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_afribooms| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|af| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.60 | 0.67 | 0.63 | 665 | | ADP | 0.76 | 0.78 | 0.77 | 1299 | | ADV | 0.74 | 0.69 | 0.72 | 523 | | AUX | 0.85 | 0.83 | 0.84 | 663 | | CCONJ | 0.71 | 0.71 | 0.71 | 380 | | DET | 0.83 | 0.70 | 0.76 | 1014 | | NOUN | 0.69 | 0.72 | 0.71 | 2025 | | NUM | 0.76 | 0.76 | 0.76 | 42 | | PART | 0.67 | 0.68 | 0.68 | 322 | | PRON | 0.87 | 0.87 | 0.87 | 794 | | PROPN | 0.82 | 0.73 | 0.77 | 156 | | PUNCT | 0.68 | 0.70 | 0.69 | 877 | | SCONJ | 0.85 | 0.85 | 0.85 | 210 | | SYM | 0.87 | 0.88 | 0.87 | 142 | | VERB | 0.69 | 0.72 | 0.70 | 889 | | X | 0.35 | 0.14 | 0.20 | 64 | | accuracy | | | 0.74 | 10065 | | macro avg | 0.73 | 0.72 | 0.72 | 10065 | | weighted avg | 0.74 | 0.74 | 0.74 | 10065 | ``` --- layout: model title: English DistilBertForQuestionAnswering model (from vitusya) author: John Snow Labs name: distilbert_qa_vitusya_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `vitusya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_vitusya_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726511045.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_vitusya_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726511045.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vitusya_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vitusya_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_vitusya").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vitusya_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vitusya/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Investment Subadvisory Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_investment_subadvisory_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, investment_subadvisory, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_investment_subadvisory_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `investment-subadvisory-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `investment-subadvisory-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_investment_subadvisory_agreement_en_1.0.0_3.0_1668115303552.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_investment_subadvisory_agreement_en_1.0.0_3.0_1668115303552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[investment-subadvisory-agreement]| |[other]| |[other]| |[investment-subadvisory-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_investment_subadvisory_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support investment-subadvisory-agreement 1.00 0.98 0.99 42 other 0.99 1.00 0.99 66 accuracy - - 0.99 108 macro-avg 0.99 0.99 0.99 108 weighted-avg 0.99 0.99 0.99 108 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Walterchamy) author: John Snow Labs name: distilbert_qa_walterchamy_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Walterchamy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_walterchamy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769525196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_walterchamy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769525196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_walterchamy_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_walterchamy_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_walterchamy_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Walterchamy/distilbert-base-uncased-finetuned-squad --- layout: model title: Medically Sound Suggestion Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_vop_sound_medical date: 2023-06-13 tags: [licensed, clinical, classification, en, vop, tensorflow] task: Text Classification language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT based](https://github.com/dmis-lab/biobert) classifier is meant to identify whether the suggestion that is mentioned in the text is medically sound. ## Predicted Entities `True`, `False` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_sound_medical_en_4.4.3_3.0_1686673701807.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_sound_medical_en_4.4.3_3.0_1686673701807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------+-------+ |text |result | +----------------------------------------------------------------------------------+-------+ |I had a lung surgery for emphyema and after surgery my xray showing some recovery.|[True] | |I was advised to put honey on a burned skin. |[False]| +----------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_sound_medical| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset “Hello,I’m 20 year old girl. I’m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I’m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I’m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I’m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you.” ## Benchmarking ```bash label precision recall f1-score support False 0.848564 0.752315 0.797546 432 True 0.664577 0.785185 0.719864 270 accuracy - - 0.764957 702 macro_avg 0.756570 0.768750 0.758705 702 weighted_avg 0.777800 0.764957 0.767668 702 ``` --- layout: model title: English Bert Embeddings (Base, Uncased, Unstructured) author: John Snow Labs name: bert_embeddings_bert_base_uncased_sparse_70_unstructured date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-uncased-sparse-70-unstructured` is a English model orginally trained by `Intel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_uncased_sparse_70_unstructured_en_3.4.2_3.0_1649672494464.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_uncased_sparse_70_unstructured_en_3.4.2_3.0_1649672494464.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_uncased_sparse_70_unstructured| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|228.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Intel/bert-base-uncased-sparse-70-unstructured --- layout: model title: Word2Vec Embeddings in Russian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, ru, open_source] task: Embeddings language: ru edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ru_3.4.1_3.0_1647455083959.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ru_3.4.1_3.0_1647455083959.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_asia_and_oceania_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Asia_and_Oceania]| |[Other]| |[Other]| |[Asia_and_Oceania]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_asia_and_oceania_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Asia_and_Oceania 0.92 0.91 0.92 456 Other 0.90 0.92 0.91 400 accuracy - - 0.91 856 macro-avg 0.91 0.91 0.91 856 weighted-avg 0.91 0.91 0.91 856 ``` --- layout: model title: Hindi RoBERTa Embeddings (from neuralspace-reverie) author: John Snow Labs name: roberta_embeddings_indic_transformers_hi_roberta date: 2022-04-14 tags: [roberta, embeddings, hi, open_source] task: Embeddings language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indic-transformers-hi-roberta` is a Hindi model orginally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_hi_roberta_hi_3.4.2_3.0_1649947526435.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_hi_roberta_hi_3.4.2_3.0_1649947526435.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indic_transformers_hi_roberta| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|313.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-hi-roberta - https://oscar-corpus.com/ --- layout: model title: Dutch BERT Sentence Base Cased Embedding author: John Snow Labs name: sent_bert_base_cased date: 2021-09-06 tags: [dutch, open_source, bert_sentence_embeddings, cased, nl] task: Embeddings language: nl edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BERTje is a Dutch pre-trained BERT model developed at the University of Groningen. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_nl_3.2.2_3.0_1630926264607.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_nl_3.2.2_3.0_1630926264607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "nl") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "nl") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu nlu.load("nl.embed_sentence.bert.base_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_base_cased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|nl| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/GroNLP/bert-base-dutch-cased --- layout: model title: English DebertaForQuestionAnswering model (from nbroad) author: John Snow Labs name: deberta_v3_xsmall_qa_squad2 date: 2022-06-15 tags: [open_source, deberta, question_answering, en] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DeBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deberta-v3-xsmall-squad2` is a English model originally trained by `nbroad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_v3_xsmall_qa_squad2_en_4.0.0_3.0_1655290640197.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_v3_xsmall_qa_squad2_en_4.0.0_3.0_1655290640197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DebertaForQuestionAnswering.pretrained("deberta_v3_xsmall_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DebertaForQuestionAnswering.pretrained("deberta_v3_xsmall_qa_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.deberta").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_v3_xsmall_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|252.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References https://huggingface.co/nbroad/deberta-v3-xsmall-squad2 --- layout: model title: Relation Extraction between dates and other entities (ReDL) author: John Snow Labs name: redl_oncology_temporal_biobert_wip date: 2022-09-29 tags: [licensed, clinical, oncology, en, relation_extraction, temporal] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links Date and Relative_Date extractions to clinical entities such as Test or Cancer_Dx. ## Predicted Entities `is_date_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_temporal_biobert_wip_en_4.1.0_3.0_1664456191667.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_temporal_biobert_wip_en_4.1.0_3.0_1664456191667.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Each relevant relation pair in the pipeline should include one date entity (Date or Relative_Date) and a clinical entity (such as Pathology_Test, Cancer_Dx or Chemotherapy).

## Results ```bash | chunk1 | entity1 | chunk2 | entity2 | relation | confidence | | --------------- |--------------- |---------------- |--------------- |----------- |----------- | | breast cancer | Cancer_Dx | last year | Relative_Date | is_date_of | 0.9999256 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_temporal_biobert_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 support O 0.77 0.81 0.79 302.0 is_date_of 0.82 0.78 0.80 298.0 macro-avg 0.79 0.79 0.79 NaN ``` --- layout: model title: Dutch RoBERTa Embeddings (Merged) author: John Snow Labs name: roberta_embeddings_robbertje_1_gb_merged date: 2022-04-14 tags: [roberta, embeddings, nl, open_source] task: Embeddings language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `robbertje-1-gb-merged` is a Dutch model orginally trained by `DTAI-KULeuven`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robbertje_1_gb_merged_nl_3.4.2_3.0_1649949144654.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robbertje_1_gb_merged_nl_3.4.2_3.0_1649949144654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_robbertje_1_gb_merged| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|nl| |Size:|279.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/DTAI-KULeuven/robbertje-1-gb-merged - http://github.com/iPieter/robbert - http://github.com/iPieter/robbertje - https://www.clinjournal.org/clinj/article/view/131 - https://www.clin31.ugent.be - https://arxiv.org/abs/2101.05716 --- layout: model title: Part of Speech for Vietnamese author: John Snow Labs name: pos_vtb date: 2021-03-10 tags: [open_source, pos, vi] supported: true task: Part of Speech Tagging language: vi edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_vtb_vi_2.7.5_2.4_1615401332222.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_vtb_vi_2.7.5_2.4_1615401332222.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_vtb", "vi") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Thắng sẽ tìm nghề mới cho Lan .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_vtb", "vi") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("Thắng sẽ tìm nghề mới cho Lan .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Thắng sẽ tìm nghề mới cho Lan .""] token_df = nlu.load('vi.pos.vtb').predict(text) token_df ```

## Results ```bash +-------------------------------+--------------------------------------------+ |text |result | +-------------------------------+--------------------------------------------+ |Thắng sẽ tìm nghề mới cho Lan .|[NOUN, X, VERB, NOUN, ADJ, ADP, NOUN, PUNCT]| +-------------------------------+--------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_vtb| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|vi| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.58 | 0.49 | 0.53 | 738 | | ADP | 0.84 | 0.87 | 0.86 | 688 | | AUX | 0.79 | 0.95 | 0.87 | 132 | | CCONJ | 0.85 | 0.80 | 0.83 | 335 | | DET | 0.95 | 0.85 | 0.90 | 232 | | INTJ | 1.00 | 0.14 | 0.25 | 7 | | NOUN | 0.84 | 0.86 | 0.85 | 3838 | | NUM | 0.94 | 0.91 | 0.92 | 412 | | PART | 0.53 | 0.30 | 0.38 | 87 | | PROPN | 0.85 | 0.85 | 0.85 | 494 | | PUNCT | 0.97 | 0.99 | 0.98 | 1722 | | SCONJ | 0.99 | 0.98 | 0.98 | 122 | | VERB | 0.73 | 0.76 | 0.74 | 2178 | | X | 0.81 | 0.76 | 0.79 | 970 | | accuracy | | | 0.83 | 11955 | | macro avg | 0.83 | 0.75 | 0.77 | 11955 | | weighted avg | 0.83 | 0.83 | 0.83 | 11955 | ``` --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from autoevaluate) author: John Snow Labs name: distilbert_qa_autoevaluate_base_cased_led_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad` is a English model originally trained by `autoevaluate`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_autoevaluate_base_cased_led_squad_en_4.3.0_3.0_1672766463212.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_autoevaluate_base_cased_led_squad_en_4.3.0_3.0_1672766463212.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_autoevaluate_base_cased_led_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_autoevaluate_base_cased_led_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_autoevaluate_base_cased_led_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/autoevaluate/distilbert-base-cased-distilled-squad --- layout: model title: Legal Entire Agreements Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_entire_agreements_bert date: 2023-03-05 tags: [en, legal, classification, clauses, entire_agreements, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Entire_Agreements` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Entire_Agreements`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_entire_agreements_bert_en_1.0.0_3.0_1678050004746.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_entire_agreements_bert_en_1.0.0_3.0_1678050004746.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_entire_agreements_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Entire_Agreements]| |[Other]| |[Other]| |[Entire_Agreements]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_entire_agreements_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Entire_Agreements 0.99 0.98 0.98 284 Other 0.98 0.99 0.98 312 accuracy - - 0.98 596 macro-avg 0.98 0.98 0.98 596 weighted-avg 0.98 0.98 0.98 596 ``` --- layout: model title: Detect clinical events author: John Snow Labs name: ner_events_healthcare date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect clinical events like Date, Occurance, Clinical_Department and a lot more using pretrained NER model. ## Predicted Entities `OCCURRENCE`, `TREATMENT`, `TIME`, `DATE`, `PROBLEM`, `CLINICAL_DEPT`, `DURATION`, `EVIDENTIAL`, `FREQUENCY`, `TEST` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_en_3.0.0_3.0_1617260839291.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_en_3.0.0_3.0_1617260839291.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_events_healthcare", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_events_healthcare", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.events_healthcre").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_healthcare| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash entity tp fp fn total precision recall f1 DURATION 575.0 263.0 231.0 806.0 0.6862 0.7134 0.6995 PROBLEM 8067.0 2479.0 2305.0 10372.0 0.7649 0.7778 0.7713 DATE 1787.0 508.0 315.0 2102.0 0.7786 0.8501 0.8128 CLINICAL_DEPT 1804.0 393.0 338.0 2142.0 0.8211 0.8422 0.8315 OCCURRENCE 1917.0 893.0 2188.0 4105.0 0.6822 0.467 0.5544 TREATMENT 4578.0 1596.0 1817.0 6395.0 0.7415 0.7159 0.7285 FREQUENCY 145.0 46.0 213.0 358.0 0.7592 0.405 0.5282 TEST 3723.0 949.0 1113.0 4836.0 0.7969 0.7699 0.7831 EVIDENTIAL 334.0 80.0 279.0 613.0 0.8068 0.5449 0.6504 macro - - - - - - 0.60759 micro - - - - - - 0.73065 ``` --- layout: model title: Detect Clinical Entities (jsl_ner_wip_greedy_clinical) author: John Snow Labs name: jsl_ner_wip_greedy_clinical date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `Injury_or_Poisoning`, `Direction`, `Test`, `Admission_Discharge`, `Death_Entity`, `Relationship_Status`, `Duration`, `Hyperlipidemia`, `Respiration`, `Birth_Entity`, `Age`, `Family_History_Header`, `Labour_Delivery`, `BMI`, `Temperature`, `Alcohol`, `Kidney_Disease`, `Oncological`, `Medical_History_Header`, `Cerebrovascular_Disease`, `Oxygen_Therapy`, `O2_Saturation`, `Psychological_Condition`, `Heart_Disease`, `Employment`, `Obesity`, `Disease_Syndrome_Disorder`, `Pregnancy`, `ImagingFindings`, `Procedure`, `Medical_Device`, `Race_Ethnicity`, `Section_Header`, `Drug`, `Symptom`, `Treatment`, `Substance`, `Route`, `Blood_Pressure`, `Diet`, `External_body_part_or_region`, `LDL`, `VS_Finding`, `Allergen`, `EKG_Findings`, `Imaging_Technique`, `Triglycerides`, `RelativeTime`, `Gender`, `Pulse`, `Social_History_Header`, `Substance_Quantity`, `Diabetes`, `Modifier`, `Internal_organ_or_component`, `Clinical_Dept`, `Form`, `Strength`, `Fetus_NewBorn`, `RelativeDate`, `Height`, `Test_Result`, `Time`, `Frequency`, `Sexually_Active_or_Sexual_Orientation`, `Weight`, `Vaccine`, `Vital_Signs_Header`, `Communicable_Disease`, `Dosage`, `Hypertension`, `HDL`, `Overweight`, `Total_Cholesterol`, `Smoking`, `Date`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_clinical_en_3.0.0_3.0_1617206898504.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_clinical_en_3.0.0_3.0_1617206898504.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------+----------------------------+ |chunk |ner_label | +----------------------------------------------+----------------------------+ |21-day-old |Age | |Caucasian |Race_Ethnicity | |male |Gender | |for 2 days |Duration | |congestion |Symptom | |mom |Gender | |suctioning yellow discharge |Symptom | |nares |External_body_part_or_region| |she |Gender | |mild problems with his breathing while feeding|Symptom | |perioral cyanosis |Symptom | |retractions |Symptom | |One day ago |RelativeDate | |mom |Gender | |tactile temperature |Symptom | |Tylenol |Drug | |Baby |Age | |decreased p.o. intake |Symptom | |His |Gender | |20 minutes |Duration | |q.2h. |Frequency | |to 5 to 10 minutes |Duration | |his |Gender | |respiratory congestion |Symptom | |He |Gender | |tired |Symptom | |fussy |Symptom | |over the past 2 days |RelativeDate | |albuterol |Drug | |ER |Clinical_Dept | |His |Gender | |urine output has also decreased |Symptom | |he |Gender | |per 24 hours |Frequency | |he |Gender | |per 24 hours |Frequency | |Mom |Gender | |diarrhea |Symptom | |His |Gender | |bowel |Internal_organ_or_component | +----------------------------------------------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_greedy_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash entity tp fp fn total precision recall f1 VS_Finding 229.0 56.0 34.0 263.0 0.8035 0.8707 0.8358 Direction 4009.0 479.0 403.0 4412.0 0.8933 0.9087 0.9009 Female_Reproducti... 2.0 1.0 3.0 5.0 0.6667 0.4 0.5 Respiration 80.0 9.0 14.0 94.0 0.8989 0.8511 0.8743 Cerebrovascular_D... 82.0 27.0 18.0 100.0 0.7523 0.82 0.7847 not 4.0 0.0 0.0 4.0 1.0 1.0 1.0 Family_History_He... 86.0 4.0 3.0 89.0 0.9556 0.9663 0.9609 Heart_Disease 469.0 76.0 83.0 552.0 0.8606 0.8496 0.8551 ImagingFindings 68.0 38.0 75.0 143.0 0.6415 0.4755 0.5462 RelativeTime 141.0 76.0 66.0 207.0 0.6498 0.6812 0.6651 Strength 720.0 49.0 58.0 778.0 0.9363 0.9254 0.9308 Smoking 117.0 8.0 6.0 123.0 0.936 0.9512 0.9435 Medical_Device 3584.0 730.0 359.0 3943.0 0.8308 0.909 0.8681 EKG_Findings 41.0 20.0 45.0 86.0 0.6721 0.4767 0.5578 Pulse 138.0 23.0 24.0 162.0 0.8571 0.8519 0.8545 Psychological_Con... 121.0 14.0 29.0 150.0 0.8963 0.8067 0.8491 Overweight 5.0 2.0 0.0 5.0 0.7143 1.0 0.8333 Triglycerides 3.0 0.0 0.0 3.0 1.0 1.0 1.0 Obesity 49.0 6.0 4.0 53.0 0.8909 0.9245 0.9074 Admission_Discharge 325.0 30.0 2.0 327.0 0.9155 0.9939 0.9531 HDL 2.0 1.0 1.0 3.0 0.6667 0.6667 0.6667 Diabetes 118.0 13.0 7.0 125.0 0.9008 0.944 0.9219 Section_Header 3778.0 148.0 138.0 3916.0 0.9623 0.9648 0.9635 Age 617.0 52.0 47.0 664.0 0.9223 0.9292 0.9257 O2_Saturation 34.0 11.0 19.0 53.0 0.7556 0.6415 0.6939 Kidney_Disease 114.0 5.0 12.0 126.0 0.958 0.9048 0.9306 Test 2668.0 526.0 498.0 3166.0 0.8353 0.8427 0.839 Communicable_Disease 25.0 12.0 9.0 34.0 0.6757 0.7353 0.7042 Hypertension 152.0 10.0 6.0 158.0 0.9383 0.962 0.95 External_body_par... 652.0 387.0 340.0 2992.0 0.8727 0.8864 0.8795 Oxygen_Therapy 67.0 21.0 23.0 90.0 0.7614 0.7444 0.7528 Test_Result 1124.0 227.0 258.0 1382.0 0.832 0.8133 0.8225 Modifier 539.0 185.0 309.0 848.0 0.7445 0.6356 0.6858 BMI 7.0 1.0 1.0 8.0 0.875 0.875 0.875 Labour_Delivery 75.0 19.0 23.0 98.0 0.7979 0.7653 0.7813 Employment 249.0 51.0 57.0 306.0 0.83 0.8137 0.8218 Clinical_Dept 948.0 95.0 80.0 1028.0 0.9089 0.9222 0.9155 Time 36.0 7.0 7.0 43.0 0.8372 0.8372 0.8372 Procedure 3180.0 460.0 480.0 3660.0 0.8736 0.8689 0.8712 Diet 50.0 29.0 30.0 80.0 0.6329 0.625 0.6289 Oncological 478.0 46.0 50.0 528.0 0.9122 0.9053 0.9087 LDL 3.0 0.0 2.0 5.0 1.0 0.6 0.75 Symptom 6801.0 1097.0 1097.0 7898.0 0.8611 0.8611 0.8611 Temperature 109.0 12.0 7.0 116.0 0.9008 0.9397 0.9198 Vital_Signs_Header 213.0 27.0 16.0 229.0 0.8875 0.9301 0.9083 Relationship_Status 42.0 2.0 1.0 43.0 0.9545 0.9767 0.9655 Total_Cholesterol 10.0 4.0 5.0 15.0 0.7143 0.6667 0.6897 Blood_Pressure 167.0 22.0 23.0 190.0 0.8836 0.8789 0.8813 Injury_or_Poisoning 510.0 83.0 111.0 621.0 0.86 0.8213 0.8402 Drug_Ingredient 1698.0 160.0 158.0 1856.0 0.9139 0.9149 0.9144 Treatment 156.0 40.0 54.0 210.0 0.7959 0.7429 0.7685 Assertion_SocialD... 4.0 0.0 6.0 10.0 1.0 0.4 0.5714 Pregnancy 100.0 45.0 41.0 141.0 0.6897 0.7092 0.6993 Vaccine 13.0 3.0 6.0 19.0 0.8125 0.6842 0.7429 Disease_Syndrome_... 2861.0 452.0 376.0 3237.0 0.8636 0.8838 0.8736 Height 25.0 8.0 9.0 34.0 0.7576 0.7353 0.7463 Frequency 650.0 157.0 148.0 798.0 0.8055 0.8145 0.81 Route 872.0 83.0 85.0 957.0 0.9131 0.9112 0.9121 Death_Entity 49.0 7.0 6.0 55.0 0.875 0.8909 0.8829 Duration 367.0 132.0 95.0 462.0 0.7355 0.7944 0.7638 Internal_organ_or... 6532.0 1016.0 987.0 7519.0 0.8654 0.8687 0.8671 Alcohol 79.0 20.0 12.0 91.0 0.798 0.8681 0.8316 Date 515.0 19.0 19.0 534.0 0.9644 0.9644 0.9644 Hyperlipidemia 47.0 2.0 1.0 48.0 0.9592 0.9792 0.9691 Social_History_He... 89.0 9.0 4.0 93.0 0.9082 0.957 0.9319 Race_Ethnicity 113.0 0.0 3.0 116.0 1.0 0.9741 0.9869 Imaging_Technique 47.0 31.0 30.0 77.0 0.6026 0.6104 0.6065 Drug_BrandName 963.0 72.0 79.0 1042.0 0.9304 0.9242 0.9273 RelativeDate 553.0 128.0 121.0 674.0 0.812 0.8205 0.8162 Gender 6043.0 59.0 87.0 6130.0 0.9903 0.9858 0.9881 Form 227.0 35.0 47.0 274.0 0.8664 0.8285 0.847 Dosage 279.0 42.0 62.0 341.0 0.8692 0.8182 0.8429 Medical_History_H... 117.0 4.0 11.0 128.0 0.9669 0.9141 0.9398 Substance 59.0 16.0 16.0 75.0 0.7867 0.7867 0.7867 Weight 85.0 19.0 21.0 106.0 0.8173 0.8019 0.8095 macro - - - - - - 0.7286 micro - - - - - - 0.8715 ``` --- layout: model title: English BertForMaskedLM Cased model (from anferico) author: John Snow Labs name: bert_embeddings_for_patents date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-for-patents` is a English model originally trained by `anferico`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_for_patents_en_4.2.4_3.0_1670019529456.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_for_patents_en_4.2.4_3.0_1670019529456.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_for_patents","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_for_patents","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_for_patents| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/anferico/bert-for-patents - https://cloud.google.com/blog/products/ai-machine-learning/how-ai-improves-patent-analysis - https://services.google.com/fh/files/blogs/bert_for_patents_white_paper.pdf - https://github.com/google/patents-public-data/blob/master/models/BERT%20for%20Patents.md - https://github.com/ec-jrc/Patents4IPPC - https://picampus-school.com/ - https://ec.europa.eu/jrc/en --- layout: model title: Italian Bert Embeddings (from bullmount) author: John Snow Labs name: bert_embeddings_hseBert_it_cased date: 2022-04-11 tags: [bert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `hseBert-it-cased` is a Italian model orginally trained by `bullmount`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_hseBert_it_cased_it_3.4.2_3.0_1649676875956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_hseBert_it_cased_it_3.4.2_3.0_1649676875956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_hseBert_it_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|412.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/bullmount/hseBert-it-cased --- layout: model title: Castilian, Spanish BertForQuestionAnswering model (from CenIA) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-qa-tar` is a Castilian, Spanish model orginally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar_es_4.0.0_3.0_1654180441819.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar_es_4.0.0_3.0_1654180441819.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.bert.base_cased.by_CenIA").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_tar| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/bert-base-spanish-wwm-cased-finetuned-qa-tar --- layout: model title: Fast Neural Machine Translation Model from Niuean to English author: John Snow Labs name: opus_mt_niu_en date: 2020-12-29 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, niu, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `niu` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_niu_en_xx_2.7.0_2.4_1609254471115.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_niu_en_xx_2.7.0_2.4_1609254471115.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_niu_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for SNOMED (sbiobertresolve_snomed_drug) author: John Snow Labs name: sbiobertresolve_snomed_drug date: 2022-01-18 tags: [licensed, snomed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps detected drug entities to SNOMED codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. ## Predicted Entities `SNOMED Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_drug_en_3.3.4_2.4_1642534694043.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_drug_en_3.3.4_2.4_1642534694043.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+------+-----------------+-----------------+------------------------------------------------------------+------------------------------------------------------------+ | ner_chunk|entity| snomed_code| resolved_text| all_k_results| all_k_resolutions| +-----------------+------+-----------------+-----------------+------------------------------------------------------------+------------------------------------------------------------+ | Fragmin| DRUG| 9487801000001106| Fragmin|9487801000001106:::130752006:::28999000:::953500100000110...|Fragmin:::Fragilysin:::Fusarin:::Femulen:::Fumonisin:::Fr...| | OxyContin| DRUG| 9296001000001100| OxyCONTIN|9296001000001100:::373470001:::230091000001108:::55452001...|OxyCONTIN:::Oxychlorosene:::Oxyargin:::oxyCODONE:::Oxymor...| | folic acid| DRUG| 63718003| Folic acid|63718003:::6247001:::226316008:::432165000:::438451000124...|Folic acid:::Folic acid-containing product:::Folic acid s...| | levothyroxine| DRUG|10071011000001106| Levothyroxine|10071011000001106:::710809001:::768532006:::126202002:::7...|Levothyroxine:::Levothyroxine (substance):::Levothyroxine...| | Avandia| DRUG| 9217601000001109| avandia|9217601000001109:::9217501000001105:::12226401000001108::...|avandia:::avandamet:::Anatera:::Intanza:::Avamys:::Aragam...| | aspirin| DRUG| 387458008| Aspirin|387458008:::7947003:::5145711000001107:::426365001:::4125...|Aspirin:::Aspirin-containing product:::Aspirin powder:::A...| | Neurontin| DRUG| 9461401000001102| neurontin|9461401000001102:::130694004:::86822004:::952840100000110...|neurontin:::Neurolysin:::Neurine (substance):::Nebilet:::...| |magnesium citrate| DRUG| 12495006|Magnesium citrate|12495006:::387401007:::21691008:::15531411000001106:::408...|Magnesium citrate:::Magnesium carbonate:::Magnesium trisi...| | insulin| DRUG| 67866001| Insulin|67866001:::325072002:::414515005:::39487003:::411530000::...|Insulin:::Insulin aspart:::Insulin detemir:::Insulin-cont...| +-----------------+------+-----------------+-----------------+------------------------------------------------------------+------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_drug| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[snomed_code]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Dependencies:|ner_posology| ## Data Source Trained on `SNOMED` code dataset with `sbiobert_base_cased_mli` sentence embeddings. --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding ICDO Codes author: John Snow Labs name: snomed_icdo_mapping date: 2023-06-13 tags: [en, licensed, clinical, pipeline, chunk_mapping, snomed, icdo] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_icdo_mapper` model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_icdo_mapping_en_4.4.4_3.2_1686665539427.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_icdo_mapping_en_4.4.4_3.2_1686665539427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("snomed_icdo_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(10376009 2026006 26638004) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("snomed_icdo_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(10376009 2026006 26638004) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.snomed_to_icdo.pipe").predict("""Put your text here.""") ```

## Results ```bash Results | | snomed_code | icdo_code | |---:|:------------------------------|:-------------------------| | 0 | 10376009 | 2026006 | 26638004 | 8050/2 | 9014/0 | 8322/0 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_icdo_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|212.8 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English BertForQuestionAnswering Cased model (from chanifrusydi) author: John Snow Labs name: bert_qa_chanifrusydi_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `chanifrusydi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_chanifrusydi_finetuned_squad_en_4.0.0_3.0_1657186373688.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_chanifrusydi_finetuned_squad_en_4.0.0_3.0_1657186373688.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_chanifrusydi_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_chanifrusydi_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_chanifrusydi_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/chanifrusydi/bert-finetuned-squad --- layout: model title: Smaller BERT Sentence Embeddings (L-2_H-512_A-8) author: John Snow Labs name: sent_small_bert_L2_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L2_512_en_2.6.0_2.4_1598350526043.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L2_512_en_2.6.0_2.4_1598350526043.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L2_512", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L2_512", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L2_512').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_small_bert_L2_512_embeddings I hate cancer [0.015892572700977325, 0.21051561832427979, 0.... Antibiotics aren't painkiller [-0.2904765009880066, 0.21515187621116638, 0.1... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L2_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1 --- layout: model title: Sentence Detection in Sindhi Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [sd, open_source, sentence_detection] task: Sentence Detection language: sd edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_sd_3.2.0_3.0_1630337452693.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_sd_3.2.0_3.0_1630337452693.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "sd") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""readingولي رھيا آھن ھڪڙو وڏو ذريعو انگريزي پڙھڻ جا پيراگراف؟ توھان صحيح ھن place تي آيا آھيو. هڪ تازي تحقيق مطابق ا today's جي نوجوانن ۾ پڙهڻ جي عادت تيزيءَ سان گهٽجي رهي آهي. اھي نٿا ڏئي سگھن انگريزي ڏنل پيراگراف تي ڪجھ سيڪنڊن کان و forيڪ لاءِ. پڻ ، پڙهڻ هو ۽ آهي هڪ لازمي حصو س allني مقابلي واري امتحانن جو. تنھنڪري ، توھان پنھنجي پڙھڻ جي صلاحيتن کي ڪيئن بھتر ڪريو ٿا؟ ھن سوال جو جواب اصل ۾ ھڪڙو questionيو سوال آھي: پڙھڻ جي صلاحيتن جو استعمال ا آھي؟ پڙهڻ جو بنيادي مقصد آهي ’احساس ڪرڻ‘.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "sd") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("readingولي رھيا آھن ھڪڙو وڏو ذريعو انگريزي پڙھڻ جا پيراگراف؟ توھان صحيح ھن place تي آيا آھيو. هڪ تازي تحقيق مطابق ا today's جي نوجوانن ۾ پڙهڻ جي عادت تيزيءَ سان گهٽجي رهي آهي. اھي نٿا ڏئي سگھن انگريزي ڏنل پيراگراف تي ڪجھ سيڪنڊن کان و forيڪ لاءِ. پڻ ، پڙهڻ هو ۽ آهي هڪ لازمي حصو س allني مقابلي واري امتحانن جو. تنھنڪري ، توھان پنھنجي پڙھڻ جي صلاحيتن کي ڪيئن بھتر ڪريو ٿا؟ ھن سوال جو جواب اصل ۾ ھڪڙو questionيو سوال آھي: پڙھڻ جي صلاحيتن جو استعمال ا آھي؟ پڙهڻ جو بنيادي مقصد آهي ’احساس ڪرڻ‘.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load('sd.sentence_detector').predict("readingولي رھيا آھن ھڪڙو وڏو ذريعو انگريزي پڙھڻ جا پيراگراف؟ توھان صحيح ھن place تي آيا آھيو. هڪ تازي تحقيق مطابق ا today's جي نوجوانن ۾ پڙهڻ جي عادت تيزيءَ سان گهٽجي رهي آهي. اھي نٿا ڏئي سگھن انگريزي ڏنل پيراگراف تي ڪجھ سيڪنڊن کان و forيڪ لاءِ. پڻ ، پڙهڻ هو ۽ آهي هڪ لازمي حصو س allني مقابلي واري امتحانن جو. تنھنڪري ، توھان پنھنجي پڙھڻ جي صلاحيتن کي ڪيئن بھتر ڪريو ٿا؟ ھن سوال جو جواب اصل ۾ ھڪڙو questionيو سوال آھي: پڙھڻ جي صلاحيتن جو استعمال ا آھي؟ پڙهڻ جو بنيادي مقصد آهي ’احساس ڪرڻ‘.", output_level ='sentence') ```

## Results ```bash +--------------------------------------------------------------------------------------------------------------+ |result | +--------------------------------------------------------------------------------------------------------------+ |[readingولي رھيا آھن ھڪڙو وڏو ذريعو انگريزي پڙھڻ جا پيراگراف؟ توھان صحيح ھن place تي آيا آھيو.] | |[هڪ تازي تحقيق مطابق ا today's جي نوجوانن ۾ پڙهڻ جي عادت تيزيءَ سان گهٽجي رهي آهي.] | |[اھي نٿا ڏئي سگھن انگريزي ڏنل پيراگراف تي ڪجھ سيڪنڊن کان و forيڪ لاءِ.] | |[پڻ ، پڙهڻ هو ۽ آهي هڪ لازمي حصو س allني مقابلي واري امتحانن جو.] | |[تنھنڪري ، توھان پنھنجي پڙھڻ جي صلاحيتن کي ڪيئن بھتر ڪريو ٿا؟ ھن سوال جو جواب اصل ۾ ھڪڙو questionيو سوال آھي:]| |[پڙھڻ جي صلاحيتن جو استعمال ا آھي؟ پڙهڻ جو بنيادي مقصد آهي ’احساس ڪرڻ‘.] | +--------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|sd| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from avioo1) author: John Snow Labs name: distilbert_qa_avioo1_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `avioo1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_avioo1_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725061133.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_avioo1_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725061133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_avioo1_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_avioo1_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_avioo1").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_avioo1_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/avioo1/distilbert-base-uncased-finetuned-squad --- layout: model title: Telugu Bert Embeddings (from monsoon-nlp) author: John Snow Labs name: bert_embeddings_muril_adapted_local date: 2022-04-11 tags: [bert, embeddings, te, open_source] task: Embeddings language: te edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `muril-adapted-local` is a Telugu model orginally trained by `monsoon-nlp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_te_3.4.2_3.0_1649675347372.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_te_3.4.2_3.0_1649675347372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_muril_adapted_local| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|te| |Size:|888.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/monsoon-nlp/muril-adapted-local - https://tfhub.dev/google/MuRIL/1 --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-hausa-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili_sw_4.1.0_3.0_1659353949038.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili_sw_4.1.0_3.0_1659353949038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_hausa_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-hausa-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Legal Subscription Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_subscription_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, subscription, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_subscription_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `subscription-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `subscription-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subscription_agreement_bert_en_1.0.0_3.0_1669372100379.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subscription_agreement_bert_en_1.0.0_3.0_1669372100379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_subscription_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[subscription-agreement]| |[other]| |[other]| |[subscription-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_subscription_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.97 0.95 65 subscription-agreement 0.94 0.89 0.91 35 accuracy - - 0.94 100 macro-avg 0.94 0.93 0.93 100 weighted-avg 0.94 0.94 0.94 100 ``` --- layout: model title: Legal No solicitation Clause Binary Classifier author: John Snow Labs name: legclf_no_solicitation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-solicitation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-solicitation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_solicitation_clause_en_1.0.0_3.2_1660123765936.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_solicitation_clause_en_1.0.0_3.2_1660123765936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_solicitation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-solicitation]| |[other]| |[other]| |[no-solicitation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_solicitation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-solicitation 0.93 0.96 0.94 26 other 0.98 0.96 0.97 46 accuracy - - 0.96 72 macro-avg 0.95 0.96 0.96 72 weighted-avg 0.96 0.96 0.96 72 ``` --- layout: model title: Detect entities related to road traffic author: John Snow Labs name: ner_traffic date: 2021-04-01 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect entities related to road traffic using pretrained NER model. ## Predicted Entities `ORGANIZATION_COMPANY`, `DISASTER_TYPE`, `TIME`, `TRIGGER`, `DATE`, `PERSON`, `LOCATION_STOP`, `ORGANIZATION`, `DISTANCE`, `LOCATION_STREET`, `NUMBER`, `DURATION`, `ORG_POSITION`, `LOCATION_ROUTE`, `LOCATION`, `LOCATION_CITY` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_TRAFFIC_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_traffic_de_3.0.0_3.0_1617260858901.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_traffic_de_3.0.0_3.0_1617260858901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_german = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_traffic", "de", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_german, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_german = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_traffic", "de", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_german, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("de.med_ner.traffic").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_traffic| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| ## Benchmarking ```bash entity tp fp fn total precision recall f1 DURATION 113.0 34.0 94.0 207.0 0.7687 0.5459 0.6384 ORGANIZATION_COMPANY 667.0 324.0 515.0 1182.0 0.6731 0.5643 0.6139 LOCATION_CITY 441.0 137.0 166.0 607.0 0.763 0.7265 0.7443 LOCATION_ROUTE 132.0 30.0 61.0 193.0 0.8148 0.6839 0.7437 DATE 730.0 81.0 168.0 898.0 0.9001 0.8129 0.8543 PERSON 422.0 84.0 174.0 596.0 0.834 0.7081 0.7659 LOCATION_STREET 132.0 12.0 99.0 231.0 0.9167 0.5714 0.704 LOCATION 697.0 94.0 359.0 1056.0 0.8812 0.66 0.7547 TIME 266.0 34.0 45.0 311.0 0.8867 0.8553 0.8707 TRIGGER 187.0 34.0 192.0 379.0 0.8462 0.4934 0.6233 DISTANCE 99.0 0.0 16.0 115.0 1.0 0.8609 0.9252 NUMBER 608.0 147.0 189.0 797.0 0.8053 0.7629 0.7835 LOCATION_STOP 403.0 53.0 77.0 480.0 0.8838 0.8396 0.8611 macro - - - - - - 0.6528 micro - - - - - - 0.7261 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Modified_pubmed_clinical date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Modified_pubmed_clinical` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Modified_pubmed_clinical_en_4.0.0_3.0_1657108848432.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Modified_pubmed_clinical_en_4.0.0_3.0_1657108848432.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Modified_pubmed_clinical| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|407.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Modified_pubmed_clinical --- layout: model title: Turkish BertForTokenClassification Cased model (from busecarik) author: John Snow Labs name: bert_token_classifier_loodos_sunlp_ner_turkish date: 2022-11-30 tags: [tr, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: tr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-loodos-sunlp-ner-turkish` is a Turkish model originally trained by `busecarik`. ## Predicted Entities `PRODUCT`, `TIME`, `MONEY`, `ORGANIZATION`, `LOCATION`, `TVSHOW`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_loodos_sunlp_ner_turkish_tr_4.2.4_3.0_1669815349144.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_loodos_sunlp_ner_turkish_tr_4.2.4_3.0_1669815349144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_loodos_sunlp_ner_turkish","tr") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_loodos_sunlp_ner_turkish","tr") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_loodos_sunlp_ner_turkish| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|tr| |Size:|412.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/busecarik/bert-loodos-sunlp-ner-turkish - https://github.com/SU-NLP/SUNLP-Twitter-NER-Dataset --- layout: model title: Legal Transactions With Affiliates Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_transactions_with_affiliates_bert date: 2023-03-05 tags: [en, legal, classification, clauses, transactions_with_affiliates, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Transactions_With_Affiliates` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Transactions_With_Affiliates`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_transactions_with_affiliates_bert_en_1.0.0_3.0_1678050557126.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_transactions_with_affiliates_bert_en_1.0.0_3.0_1678050557126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_transactions_with_affiliates_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Transactions_With_Affiliates]| |[Other]| |[Other]| |[Transactions_With_Affiliates]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_transactions_with_affiliates_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 1.00 0.97 0.98 61 Transactions_With_Affiliates 0.95 1.00 0.98 42 accuracy - - 0.98 103 macro-avg 0.98 0.98 0.98 103 weighted-avg 0.98 0.98 0.98 103 ``` --- layout: model title: Extract Temporal Entities from Voice of the Patient Documents (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_temporal_emb_clinical_medium date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, temporal] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts temporal references from the documents transferred from the patient’s own sentences. ## Predicted Entities `DateTime`, `Frequency`, `Duration` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_temporal_emb_clinical_medium_en_4.4.3_3.0_1686076464979.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_temporal_emb_clinical_medium_en_4.4.3_3.0_1686076464979.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------|:------------| | last month | DateTime | | yesterday | DateTime | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_temporal_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_medium| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 DateTime 3954 470 448 4402 0.89 0.90 0.90 Frequency 921 190 158 1079 0.83 0.85 0.84 Duration 1952 362 358 2310 0.84 0.85 0.84 macro_avg 6827 1022 964 7791 0.85 0.87 0.86 micro_avg 6827 1022 964 7791 0.87 0.88 0.87 ``` --- layout: model title: Arabic Bert Embeddings (Base) author: John Snow Labs name: bert_embeddings_bert_base_arabic date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic` is a Arabic model orginally trained by `asafaya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_ar_3.4.2_3.0_1649677068712.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_ar_3.4.2_3.0_1649677068712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|414.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/asafaya/bert-base-arabic - https://traces1.inria.fr/oscar/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/bert - https://www.tensorflow.org/tfrc - https://github.com/alisafaya/Arabic-BERT --- layout: model title: Tagalog Electra Embeddings (from jcblaise) author: John Snow Labs name: electra_embeddings_electra_tagalog_small_cased_generator date: 2022-05-17 tags: [tl, open_source, electra, embeddings] task: Embeddings language: tl edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-tagalog-small-cased-generator` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_small_cased_generator_tl_3.4.4_3.0_1652786760112.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_small_cased_generator_tl_3.4.4_3.0_1652786760112.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_tagalog_small_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tl| |Size:|18.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/jcblaise/electra-tagalog-small-cased-generator - https://blaisecruz.com --- layout: model title: Fast Neural Machine Translation Model from English to Malayo-Polynesian Languages author: John Snow Labs name: opus_mt_en_poz date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, poz, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `poz` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_poz_xx_2.7.0_2.4_1609168000682.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_poz_xx_2.7.0_2.4_1609168000682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_poz| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Reporting Clause Binary Classifier author: John Snow Labs name: legclf_reporting_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `reporting` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `reporting` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_reporting_clause_en_1.0.0_3.2_1660123939267.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_reporting_clause_en_1.0.0_3.2_1660123939267.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_reporting_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[reporting]| |[other]| |[other]| |[reporting]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_reporting_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.93 0.93 0.93 154 reporting 0.86 0.87 0.87 78 accuracy - - 0.91 232 macro-avg 0.90 0.90 0.90 232 weighted-avg 0.91 0.91 0.91 232 ``` --- layout: model title: English RobertaForQuestionAnswering (from deepset) author: John Snow Labs name: roberta_qa_roberta_base_squad2_covid date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-covid` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_covid_en_4.0.0_3.0_1655735153729.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_covid_en_4.0.0_3.0_1655735153729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_squad2_covid","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_squad2_covid","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_squad2_covid| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/roberta-base-squad2-covid - https://www.linkedin.com/company/deepset-ai/ - https://github.com/deepset-ai/COVID-QA/blob/master/data/question-answering/200423_covidQA.json - https://haystack.deepset.ai/community/join - https://deepset.ai/german-bert - https://github.com/deepset-ai/FARM - http://www.deepset.ai/jobs - https://twitter.com/deepset_ai - https://github.com/deepset-ai/haystack/discussions - https://github.com/deepset-ai/haystack/ - https://deepset.ai - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM/blob/master/examples/question_answering_crossvalidation.py --- layout: model title: ChunkResolver Loinc Clinical author: John Snow Labs name: chunkresolve_loinc_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities LOINC Codes with ``clinical_embeddings``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_LOINC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_loinc_clinical_en_3.0.0_3.0_1617355407030.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_loinc_clinical_en_3.0.0_3.0_1617355407030.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... loinc_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_loinc_clinical", "en", "clinical/models") \ .setInputCols(["token", "chunk_embeddings"]) \ .setOutputCol("loinc_code") \ .setDistanceFunction("COSINE") \ .setNeighbours(5) pipeline_loinc = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, loinc_resolver]) data = spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting."""]]).toDF("text") model = pipeline_loinc.fit(data) results = model.transform(data) ``` ```scala ... val loinc_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_loinc_clinical", "en", "clinical/models") .setInputCols(Array("token", "chunk_embeddings")) .setOutputCol("loinc_code") .setDistanceFunction("COSINE") .setNeighbours(5) val pipeline_loinc = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, loinc_resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting.").toDF("text") val result = pipeline_loinc.fit(data).transform(data) ```

## Results ```bash Chunk loinc-Code 0 gestational diabetes mellitus 44877-9 1 type two diabetes mellitus 44877-9 2 T2DM 93692-2 3 prior episode of HTG-induced pancreatitis 85695-5 4 associated with an acute hepatitis 24363-4 5 obesity with a body mass index 47278-7 6 BMI) of 33.5 kg/m2 47214-2 7 polyuria 35234-4 8 polydipsia 25541-4 9 poor appetite 50056-1 10 vomiting 34175-0 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_loinc_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[loinc]| |Language:|en| --- layout: model title: English image_classifier_vit_housing_categories ViTForImageClassification from Albe author: John Snow Labs name: image_classifier_vit_housing_categories date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_housing_categories` is a English model originally trained by Albe. ## Predicted Entities `tree house`, `yurt`, `caravan`, `farm`, `castle` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_housing_categories_en_4.1.0_3.0_1660166778182.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_housing_categories_en_4.1.0_3.0_1660166778182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_housing_categories", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_housing_categories", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_housing_categories| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect Clinical Entities author: John Snow Labs name: ner_jsl_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_biobert](https://nlp.johnsnowlabs.com/2021/09/05/ner_jsl_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_biobert_pipeline_en_3.4.1_3.0_1647869212989.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_biobert_pipeline_en_3.4.1_3.0_1647869212989.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_jsl_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("ner_jsl_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_macbert_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|384.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-macbert-base - https://github.com/ymcui/MacBERT/blob/master/LICENSE - https://2020.emnlp.org - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://github.com/chatopera/Synonyms - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 --- layout: model title: Legal Exhibits Clause Binary Classifier author: John Snow Labs name: legclf_exhibits_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `exhibits` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `exhibits` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_exhibits_clause_en_1.0.0_3.2_1660123526366.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_exhibits_clause_en_1.0.0_3.2_1660123526366.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_exhibits_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[exhibits]| |[other]| |[other]| |[exhibits]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_exhibits_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support exhibits 0.95 0.97 0.96 59 other 0.98 0.97 0.97 96 accuracy - - 0.97 155 macro-avg 0.96 0.97 0.97 155 weighted-avg 0.97 0.97 0.97 155 ``` --- layout: model title: Vietnamese Deberta Embeddings model (from binhquoc) author: John Snow Labs name: deberta_embeddings_vie_small date: 2023-03-12 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, vie, tensorflow] task: Embeddings language: vie edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `vie-deberta-small` is a Vietnamese model originally trained by `binhquoc`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_vie_small_vie_4.3.1_3.0_1678626638418.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_vie_small_vie_4.3.1_3.0_1678626638418.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_vie_small","vie") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_vie_small","vie") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_vie_small| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|vie| |Size:|278.0 MB| |Case sensitive:|false| ## References https://huggingface.co/binhquoc/vie-deberta-small --- layout: model title: Legal Jurisdictions Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_jurisdictions_bert date: 2023-03-05 tags: [en, legal, classification, clauses, jurisdictions, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Jurisdictions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Jurisdictions`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_jurisdictions_bert_en_1.0.0_3.0_1678050016981.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_jurisdictions_bert_en_1.0.0_3.0_1678050016981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_jurisdictions_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Jurisdictions]| |[Other]| |[Other]| |[Jurisdictions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_jurisdictions_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Jurisdictions 0.86 1.00 0.93 19 Other 1.00 0.91 0.95 32 accuracy - - 0.94 51 macro-avg 0.93 0.95 0.94 51 weighted-avg 0.95 0.94 0.94 51 ``` --- layout: model title: English Bert Embeddings (Uncased) author: John Snow Labs name: bert_embeddings_false_positives_scancode_bert_base_uncased_L8_1 date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `false-positives-scancode-bert-base-uncased-L8-1` is a English model orginally trained by `ayansinha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_false_positives_scancode_bert_base_uncased_L8_1_en_3.4.2_3.0_1649672624525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_false_positives_scancode_bert_base_uncased_L8_1_en_3.4.2_3.0_1649672624525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_false_positives_scancode_bert_base_uncased_L8_1| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|410.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/ayansinha/false-positives-scancode-bert-base-uncased-L8-1 - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer#quickstart---local-machine - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py --- layout: model title: French CamemBert Embeddings (from ysharma) author: John Snow Labs name: camembert_embeddings_ysharma_generic_model_2 date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model-2` is a French model orginally trained by `ysharma`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ysharma_generic_model_2_fr_3.4.4_3.0_1653991037086.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ysharma_generic_model_2_fr_3.4.4_3.0_1653991037086.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_ysharma_generic_model_2| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/ysharma/dummy-model-2 --- layout: model title: Multilingual BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_multi_cased_finetuned_xquadv1 date: 2022-06-02 tags: [en, es, de, el, ru, tr, ar, vi, th, zh, hi, open_source, question_answering, bert, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-cased-finetuned-xquadv1` is a Multilingual model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finetuned_xquadv1_xx_4.0.0_3.0_1654184515717.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finetuned_xquadv1_xx_4.0.0_3.0_1654184515717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_multi_cased_finetuned_xquadv1","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_multi_cased_finetuned_xquadv1","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.xquad.bert.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_multi_cased_finetuned_xquadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-multi-cased-finetuned-xquadv1 - https://github.com/google-research/bert/blob/master/multilingual.md - https://twitter.com/mrm8488 - https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl - https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Try_mrm8488_xquad_finetuned_model.ipynb - https://github.com/fxsjy/jieba - https://github.com/deepmind/xquad --- layout: model title: Legal Deposit Of Redemption Price Clause Binary Classifier author: John Snow Labs name: legclf_deposit_of_redemption_price_clause date: 2023-01-27 tags: [en, legal, classification, deposit, redemption, price, clauses, deposit_of_redemption_price, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `deposit-of-redemption-price` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `deposit-of-redemption-price`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_deposit_of_redemption_price_clause_en_1.0.0_3.0_1674820606738.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_deposit_of_redemption_price_clause_en_1.0.0_3.0_1674820606738.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_deposit_of_redemption_price_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[deposit-of-redemption-price]| |[other]| |[other]| |[deposit-of-redemption-price]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_deposit_of_redemption_price_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support deposit-of-redemption-price 0.96 1.00 0.98 22 other 1.00 0.97 0.99 38 accuracy - - 0.98 60 macro-avg 0.98 0.99 0.98 60 weighted-avg 0.98 0.98 0.98 60 ``` --- layout: model title: English BertForMaskedLM Base Cased model (from ayansinha) author: John Snow Labs name: bert_embeddings_lic_class_scancode_base_cased_l32_1 date: 2022-12-06 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `lic-class-scancode-bert-base-cased-L32-1` is a English model originally trained by `ayansinha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_lic_class_scancode_base_cased_l32_1_en_4.2.4_3.0_1670326834348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_lic_class_scancode_base_cased_l32_1_en_4.2.4_3.0_1670326834348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_lic_class_scancode_base_cased_l32_1","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_lic_class_scancode_base_cased_l32_1","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_lic_class_scancode_base_cased_l32_1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/ayansinha/lic-class-scancode-bert-base-cased-L32-1 - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer#quickstart---local-machine - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py --- layout: model title: English BertForQuestionAnswering model (from krinal214) author: John Snow Labs name: bert_qa_mBERT_all_ty_SQen_SQ20_1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBERT_all_ty_SQen_SQ20_1` is a English model orginally trained by `krinal214`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mBERT_all_ty_SQen_SQ20_1_en_4.0.0_3.0_1654188214197.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mBERT_all_ty_SQen_SQ20_1_en_4.0.0_3.0_1654188214197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mBERT_all_ty_SQen_SQ20_1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_mBERT_all_ty_SQen_SQ20_1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.multi_lingual_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mBERT_all_ty_SQen_SQ20_1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/krinal214/mBERT_all_ty_SQen_SQ20_1 --- layout: model title: English RobertaForQuestionAnswering Cased model (from skandaonsolve) author: John Snow Labs name: roberta_qa_finetuned_timeentities2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-timeentities2` is a English model originally trained by `skandaonsolve`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities2_en_4.3.0_3.0_1674220671794.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities2_en_4.3.0_3.0_1674220671794.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_timeentities2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|465.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-timeentities2 --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_becasincentivos1 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-becasIncentivos1` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos1_es_4.3.0_3.0_1674217969589.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos1_es_4.3.0_3.0_1674217969589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos1","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos1","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_becasincentivos1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|459.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-becasIncentivos1 --- layout: model title: Detect Organism in Medical Texts author: John Snow Labs name: bert_token_classifier_ner_linnaeus_species date: 2022-07-25 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. The model detects species entities in a biomedical text ## Predicted Entities `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_linnaeus_species_en_4.0.0_3.0_1658755473753.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_linnaeus_species_en_4.0.0_3.0_1658755473753.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+-------+ |ner_chunk |label | +-------------------------+-------+ |chicken |SPECIES| |human |SPECIES| |Xenopus laevis |SPECIES| |Drosophila melanogaster |SPECIES| |Schizosaccharomyces pombe|SPECIES| +-------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_linnaeus_species| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://github.com/cambridgeltl/MTL-Bioinformatics-2016](https://github.com/cambridgeltl/MTL-Bioinformatics-2016) ## Benchmarking ```bash label precision recall f1-score support B-SPECIES 0.6391 0.9204 0.7544 1433 I-SPECIES 0.8297 0.7071 0.7635 799 micro-avg 0.6863 0.8441 0.7571 2232 macro-avg 0.7344 0.8138 0.7589 2232 weighted-avg 0.7073 0.8441 0.7576 2232 ``` --- layout: model title: Finnish asr_wav2vec2_xlsr_1b_finnish TFWav2Vec2ForCTC from aapot author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_1b_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_1b_finnish` is a Finnish model originally trained by aapot. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_1b_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_1b_finnish_fi_4.2.0_3.0_1664018763154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_1b_finnish_fi_4.2.0_3.0_1664018763154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_1b_finnish', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_1b_finnish", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_1b_finnish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|3.6 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Icelandic NER Pipeline author: John Snow Labs name: roberta_token_classifier_icelandic_ner_pipeline date: 2022-04-20 tags: [open_source, ner, token_classifier, roberta, icelandic, is] task: Named Entity Recognition language: is edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_token_classifier_icelandic_ner](https://nlp.johnsnowlabs.com/2021/12/06/roberta_token_classifier_icelandic_ner_is.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_icelandic_ner_pipeline_is_3.4.1_3.0_1650453946425.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_icelandic_ner_pipeline_is_3.4.1_3.0_1650453946425.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("roberta_token_classifier_icelandic_ner_pipeline", lang = "is") pipeline.annotate("Ég heiti Peter Fergusson. Ég hef búið í New York síðan í október 2011 og unnið hjá Tesla Motor og þénað 100K $ á ári.") ``` ```scala val pipeline = new PretrainedPipeline("roberta_token_classifier_icelandic_ner_pipeline", lang = "is") pipeline.annotate("Ég heiti Peter Fergusson. Ég hef búið í New York síðan í október 2011 og unnið hjá Tesla Motor og þénað 100K $ á ári.") ```

## Results ```bash +----------------+------------+ |chunk |ner_label | +----------------+------------+ |Peter Fergusson |Person | |New York |Location | |október 2011 |Date | |Tesla Motor |Organization| |100K $ |Money | +----------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_icelandic_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Language:|is| |Size:|457.5 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: Clinical Deidentification (English, Glove, Augmented) author: John Snow Labs name: clinical_deidentification_glove_augmented date: 2022-09-16 tags: [en, deid, deidentification, licensed, clinical, glove, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with lightweight `glove_100d` embeddings and can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR` entities. It's different to `clinical_deidentification_glove` in the way it manages PHONE and PATIENT, having apart from the NER, some rules in Contextual Parser components. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_4.1.0_3.2_1663311659491.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_4.1.0_3.2_1663311659491.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification_glove_augmented", "en", "clinical/models") deid_pipeline.annotate("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = PretrainedPipeline("clinical_deidentification_glove_augmented", "en", "clinical/models") val result = pipeline.annotate("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.glove_augmented.pipeline").predict("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ```

## Results ```bash {'masked': ['Record date : , , M.D.', 'IP: .', "The driver's license no: .", 'The SSN: and e-mail: .', 'Name : MR. # Date : .', 'PCP : , years old.', 'Record date : , : .'], 'masked_fixed_length_chars': ['Record date : ****, ****, M.D.', 'IP: ****.', "The driver's license no: ****.", 'The SSN: **** and e-mail: ****.', 'Name : **** MR. # **** Date : ****.', 'PCP : ****, **** years old.', 'Record date : ****, **** : ****.'], 'masked_with_chars': ['Record date : [********], [********], M.D.', 'IP: [************].', "The driver's license no: [******].", 'The SSN: [*******] and e-mail: [************].', 'Name : [**************] MR. # [****] Date : [******].', 'PCP : [******], ** years old.', 'Record date : [********], [***********] : [***************].'], 'ner_chunk': ['2093-01-13', 'David Hale', 'A334455B', '324598674', 'hale@gmail.com', 'Hendrickson, Ora', '719435', '01/13/93', 'Oliveira', '25', '2079-11-09', "Patient's VIN", '1HGBH41JXMN109286'], 'obfuscated': ['Record date : 2093-01-23, Dr Marshia Curling, M.D.', 'IP: 004.004.004.004.', "The driver's license no: 123XX123.", 'The SSN: SSN-089-89-9294 and e-mail: Mikey@hotmail.com.', 'Name : Stephania Chang MR. # E5881795 Date : 02-14-1983.', 'PCP : Dr Lovella Israel, 52 years old.', 'Record date : 2079-11-14, Dr Colie Carne : 3CCCC22DDDD333888.'], 'sentence': ['Record date : 2093-01-13, David Hale, M.D.', 'IP: 203.120.223.13.', "The driver's license no: A334455B.", 'The SSN: 324598674 and e-mail: hale@gmail.com.', 'Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93.', 'PCP : Oliveira, 25 years old.', "Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286."]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification_glove_augmented| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|181.3 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from Davlan) author: John Snow Labs name: xlmroberta_ner_base_sadilar date: 2022-08-01 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-sadilar-ner` is a Multilingual model originally trained by `Davlan`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_sadilar_xx_4.1.0_3.0_1659356675311.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_sadilar_xx_4.1.0_3.0_1659356675311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_sadilar","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_sadilar","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_sadilar| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|806.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Davlan/xlm-roberta-base-sadilar-ner - https://www.sadilar.org/index.php/en/ --- layout: model title: Company Name to IRS (Edgar database) author: John Snow Labs name: legel_edgar_irs date: 2022-08-30 tags: [en, legal, companies, edgar, licensed] task: Entity Resolution language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an Entity Linking / Entity Resolution model, which allows you to retrieve the IRS number of a company given its name, using SEC Edgar database. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/ER_EDGAR_CRUNCHBASE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legel_edgar_irs_en_1.0.0_3.2_1661866500067.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legel_edgar_irs_en_1.0.0_3.2_1661866500067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk") \ .setOutputCol("sentence_embeddings") resolver = legal.SentenceEntityResolverModel.pretrained("legel_edgar_irs", "en", "legal/models")\ .setInputCols(["ner_chunk", "sentence_embeddings"]) \ .setOutputCol("irs_code")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = nlp.Pipeline( stages = [ documentAssembler, embeddings, resolver]) lp = LightPipeline(pipelineModel) lp.fullAnnotate("CONTACT GOLD") ```

## Results ```bash +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ | chunk| code | all_codes| resolutions | all_distances| +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ | CONTACT GOLD | 981369960| [981369960, 271989147, 208531222, 273566922, 270348508] |[981369960, 271989147, 208531222, 273566922, 270348508] | [0.1733, 0.3700, 0.3867, 0.4103, 0.4121] | +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legel_edgar_irs| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[company_irs_number]| |Language:|en| |Size:|313.8 MB| |Case sensitive:|false| ## References In-house scrapping and postprocessing of SEC Edgar Database --- layout: model title: Translate Yapese to English Pipeline author: John Snow Labs name: translate_yap_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, yap, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `yap` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_yap_en_xx_2.7.0_2.4_1609686209390.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_yap_en_xx_2.7.0_2.4_1609686209390.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_yap_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_yap_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.yap.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_yap_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_vliegmachine ViTForImageClassification from johnnydevriese author: John Snow Labs name: image_classifier_vit_vliegmachine date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vliegmachine` is a English model originally trained by johnnydevriese. ## Predicted Entities `f117`, `f16`, `f18` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vliegmachine_en_4.1.0_3.0_1660166009742.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vliegmachine_en_4.1.0_3.0_1660166009742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vliegmachine", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vliegmachine", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vliegmachine| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English RobertaForMaskedLM Large Cased model author: John Snow Labs name: roberta_embeddings_large date: 2022-12-12 tags: [en, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_large_en_4.2.4_3.0_1670859597088.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_large_en_4.2.4_3.0_1670859597088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_large","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_large","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_large| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|847.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/roberta-large - https://arxiv.org/abs/1907.11692 - https://github.com/pytorch/fairseq/tree/master/examples/roberta - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia - https://commoncrawl.org/2016/10/news-dataset-available/ - https://github.com/jcpeterson/openwebtext - https://arxiv.org/abs/1806.02847 --- layout: model title: Pipeline to Detect Anatomical References (biobert) author: John Snow Labs name: ner_anatomy_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_anatomy_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_anatomy_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_biobert_pipeline_en_3.4.1_3.0_1647873806641.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_biobert_pipeline_en_3.4.1_3.0_1647873806641.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_anatomy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along the lateral portion of her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.") ``` ```scala val pipeline = new PretrainedPipeline("ner_anatomy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along the lateral portion of her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy_biobert.pipeline").predict("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along the lateral portion of her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""") ```

## Results ```bash +-------------------+----------------------+ |chunk |ner_label | +-------------------+----------------------+ |right |Organism_subdivision | |great |Organism_subdivision | |toe |Organism_subdivision | |skin |Organ | |Sclerae |Pathological_formation| |Extraocular muscles|Multi-tissue_structure| |Nares |Organ | |turbinates |Multi-tissue_structure| |Mucous membranes |Cell | |Abdomen |Organism_subdivision | |bowel |Organism_subdivision | |right |Organism_subdivision | |toe |Organism_subdivision | |skin |Organ | |toenails |Organism_subdivision | |foot |Organism_subdivision | |toe |Organism_subdivision | |toenails |Organism_subdivision | +-------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Translate English to Slavic languages Pipeline author: John Snow Labs name: translate_en_sla date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, sla, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `sla` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_sla_xx_2.7.0_2.4_1609687363449.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_sla_xx_2.7.0_2.4_1609687363449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_sla", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_sla", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.sla').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_sla| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Environmental matters Clause Binary Classifier author: John Snow Labs name: legclf_environmental_matters_clause date: 2022-09-28 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `environmental-matters` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `environmental-matters` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_environmental_matters_clause_en_1.0.0_3.0_1664363148554.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_environmental_matters_clause_en_1.0.0_3.0_1664363148554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_environmental_matters_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[environmental-matters]| |[other]| |[other]| |[environmental-matters]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_environmental_matters_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support environmental-matters 0.95 0.86 0.90 21 other 0.94 0.98 0.96 48 accuracy - - 0.94 69 macro-avg 0.94 0.92 0.93 69 weighted-avg 0.94 0.94 0.94 69 ``` --- layout: model title: ALBERT Embeddings (Base Uncase) author: John Snow Labs name: albert_base_uncased date: 2020-04-28 task: Embeddings language: en nav_key: models edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ALBERT is "A Lite" version of BERT, a popular unsupervised language representation learning algorithm. ALBERT uses parameter-reduction techniques that allow for large-scale configurations, overcome previous memory limitations, and achieve better behavior with respect to model degradation. The details are described in the paper "[ALBERT: A Lite BERT for Self-supervised Learning of Language Representations.](https://arxiv.org/abs/1909.11942)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_base_uncased_en_2.5.0_2.4_1588073363475.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_base_uncased_en_2.5.0_2.4_1588073363475.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = AlbertEmbeddings.pretrained("albert_base_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = AlbertEmbeddings.pretrained("albert_base_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.albert.base_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_albert_base_uncased_embeddings I [1.0153148174285889, 0.5481745600700378, -0.44... love [0.3452114760875702, -1.191628336906433, 0.423... NLP [-0.4268064796924591, -0.3819553852081299, 0.8... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_base_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/albert_base/3](https://tfhub.dev/google/albert_base/3) --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578 TFWav2Vec2ForCTC from doddle124578 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578` is a English model originally trained by doddle124578. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_en_4.2.0_3.0_1664037269433.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_en_4.2.0_3.0_1664037269433.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Detect Anatomical and Observation Entities in Chest Radiology Reports (CheXpert) author: John Snow Labs name: ner_chexpert date: 2021-09-30 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts `Anatomical` and `Observation` entities from Chest Radiology Reports. ## Predicted Entities `ANAT - Anatomy`, `OBS - Observation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_RADIOLOGY/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_en_3.3.0_3.0_1633010671460.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_en_3.3.0_3.0_1633010671460.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_chexpert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base."]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_chexpert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("""FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chexpert").predict("""FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.""") ```

## Results ```bash | | chunk | label | |---:|:-------------------------|:--------| | 0 | endotracheal tube | OBS | | 1 | Swan - Ganz catheter | OBS | | 2 | left chest | ANAT | | 3 | tube | OBS | | 4 | in place | OBS | | 5 | pneumothorax | OBS | | 6 | Mild atelectatic changes | OBS | | 7 | left base | ANAT | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chexpert| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on CheXpert dataset explain in https://arxiv.org/pdf/2106.14463.pdf. ## Benchmarking ```bash label tp fp fn prec rec f1 I-ANAT_DP 26 11 11 0.7027027 0.7027027 0.7027027 B-OBS_DP 1489 141 104 0.9134969 0.9347144 0.9239839 I-OBS_DP 16 3 54 0.84210527 0.22857143 0.35955057 B-ANAT_DP 1125 39 45 0.96649486 0.96153843 0.96401024 Macro-average 2656 194 214 0.8561999 0.70688176 0.7744088 Micro-average 2656 194 214 0.9319298 0.92543554 0.9286713 ``` --- layout: model title: Legal Waivers Clause Binary Classifier author: John Snow Labs name: legclf_waivers_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `waivers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `waivers` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_waivers_clause_en_1.0.0_3.2_1660124127062.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_waivers_clause_en_1.0.0_3.2_1660124127062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_waivers_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[waivers]| |[other]| |[other]| |[waivers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_waivers_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.98 0.97 324 waivers 0.96 0.90 0.93 128 accuracy - - 0.96 452 macro-avg 0.96 0.94 0.95 452 weighted-avg 0.96 0.96 0.96 452 ``` --- layout: model title: Detect PHI for Deidentification (Sub Entity) author: John Snow Labs name: ner_deid_subentity date: 2022-01-06 tags: [de, deid, ner, licensed] task: Named Entity Recognition language: de edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM, CNN. Deidentification NER is a Named Entity Recognition model that annotates German text to find protected health information (PHI) that may need to be deidentified. It was trained with in-house annotations and detects 12 entities. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `STREET`, `USERNAME`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_DE){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_de_3.3.4_2.4_1641460993460.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_de_3.3.4_2.4_1641460993460.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","de","clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") deid_ner = MedicalNerModel.pretrained("ner_deid_subentity", "de", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_deid_subentity_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, deid_ner, ner_converter]) data = spark.createDataFrame([["""Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhaus in Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen."""]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val deid_ner = MedicalNerModel.pretrained("ner_deid_subentity", "de", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_deid_subentity_chunk") val nlpPipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, deid_ner, ner_converter)) val data = Seq("""Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhausin Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen.""").toDS.toDF("text") val result = nlpPipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("de.med_ner.deid_subentity").predict("""Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhaus in Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen.""") ```

## Results ```bash +-------------------------+-------------------------+ |chunk |ner_deid_subentity_chunk | +-------------------------+-------------------------+ |Michael Berger |PATIENT | |12 Dezember 2018 |DATE | |St. Elisabeth-Krankenhaus|HOSPITAL | |Bad Kissingen |CITY | |Berger |PATIENT | |76 |AGE | +-------------------------+-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| |Size:|15.0 MB| ## Data Source In-house annotated dataset ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 2080.0 58.0 74.0 2154.0 0.9729 0.9656 0.9692 HOSPITAL 1598.0 4.0 0.0 1598.0 0.9975 1.0 0.9988 DATE 4047.0 7.0 2.0 4049.0 0.9983 0.9995 0.9989 ORGANIZATION 1288.0 108.0 67.0 1355.0 0.9226 0.9506 0.9364 CITY 196.0 1.0 4.0 200.0 0.9949 0.98 0.9874 STREET 124.0 1.0 4.0 128.0 0.992 0.9688 0.9802 USERNAME 45.0 0.0 0.0 45.0 1.0 1.0 1.0 PROFESSION 262.0 1.0 0.0 262.0 0.9962 1.0 0.9981 PHONE 71.0 10.0 9.0 80.0 0.8765 0.8875 0.882 COUNTRY 306.0 5.0 6.0 312.0 0.9839 0.9808 0.9823 DOCTOR 1414.0 9.0 39.0 1453.0 0.9937 0.9732 0.9833 AGE 473.0 3.0 3.0 476.0 0.9937 0.9937 0.9937 ``` --- layout: model title: English BertForMaskedLM Base Uncased model author: John Snow Labs name: bert_embeddings_base_uncased date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uncased_en_4.2.4_3.0_1670019190911.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uncased_en_4.2.4_3.0_1670019190911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_uncased","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_uncased","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/bert-base-uncased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/master/README.md - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: Fast Neural Machine Translation Model from English to Berber author: John Snow Labs name: opus_mt_en_ber date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ber, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ber` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ber_xx_2.7.0_2.4_1609169805124.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ber_xx_2.7.0_2.4_1609169805124.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ber| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Indemnification Clause Binary Classifier (md) author: John Snow Labs name: legclf_indemnification_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `indemnification` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `indemnification` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_md_en_1.0.0_3.0_1669376491781.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_md_en_1.0.0_3.0_1669376491781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_indemnification_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[indemnification]| |[other]| |[other]| |[indemnification]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnification_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support indemnification-and-contribution 0.84 0.93 0.88 28 other 0.94 0.87 0.91 39 accuracy 0.90 67 macro avg 0.89 0.90 0.89 67 weighted avg 0.90 0.90 0.90 67 ``` --- layout: model title: Recognize Entities OntoNotes pipeline - BERT Small author: John Snow Labs name: onto_recognize_entities_bert_small date: 2021-03-22 tags: [open_source, english, onto_recognize_entities_bert_small, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The onto_recognize_entities_bert_small is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_small_en_3.0.0_3.0_1616443983762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_small_en_3.0.0_3.0_1616443983762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('onto_recognize_entities_bert_small', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_small", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.ner.onto.bert.small').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:----------------------------------------------------|:-------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[0.9379079937934875,.,...]] | ['O', 'O', 'B-PERSON', 'I-PERSON', 'I-PERSON', 'O'] | ['John Snow Labs'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_small| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Swty) author: John Snow Labs name: distilbert_qa_swty_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Swty`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_swty_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769390900.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_swty_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769390900.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_swty_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_swty_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_swty_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Swty/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Mapping RxNORM Codes with Their Corresponding UMLS Codes author: John Snow Labs name: rxnorm_umls_mapping date: 2023-06-13 tags: [en, licensed, clinical, pipeline, chunk_mapping, rxnorm, umls] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `rxnorm_umls_mapper` model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_4.4.4_3.2_1686663532119.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_4.4.4_3.2_1686663532119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rxnorm_umls_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(1161611 315677) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rxnorm_umls_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(1161611 315677) ``` {:.nlu-block} ```python import nlu nlu.load("en.rxnorm.umls.mapping").predict("""Put your text here.""") ```

## Results ```bash Results | | rxnorm_code | umls_code | |---:|:-----------------|:--------------------| | 0 | 1161611 | 315677 | C3215948 | C0984912 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.9 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Legal Cancellation Clause Binary Classifier author: John Snow Labs name: legclf_cancellation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `cancellation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `cancellation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cancellation_clause_en_1.0.0_3.2_1660122195560.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cancellation_clause_en_1.0.0_3.2_1660122195560.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cancellation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[cancellation]| |[other]| |[other]| |[cancellation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cancellation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support cancellation 0.92 0.92 0.92 38 other 0.97 0.97 0.97 96 accuracy - - 0.96 134 macro-avg 0.94 0.94 0.94 134 weighted-avg 0.96 0.96 0.96 134 ``` --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_marcel TFWav2Vec2ForCTC from marcel author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_marcel` is a German model originally trained by marcel. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel_de_4.2.0_3.0_1664101959541.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel_de_4.2.0_3.0_1664101959541.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_german_by_marcel| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Czech asr_wav2vec2_large_xlsr_53_Czech TFWav2Vec2ForCTC from MehdiHosseiniMoghadam author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_Czech date: 2022-09-25 tags: [wav2vec2, cs, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: cs edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_Czech` is a Czech model originally trained by MehdiHosseiniMoghadam. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_Czech_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_Czech_cs_4.2.0_3.0_1664119968423.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_Czech_cs_4.2.0_3.0_1664119968423.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_Czech', lang = 'cs') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_Czech", lang = "cs") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_Czech| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|cs| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Hausa asr_Hausa_xlsr TFWav2Vec2ForCTC from Akashpb13 author: John Snow Labs name: asr_Hausa_xlsr date: 2022-09-26 tags: [wav2vec2, ha, audio, open_source, asr] task: Automatic Speech Recognition language: ha edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Hausa_xlsr` is a Hausa model originally trained by Akashpb13. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Hausa_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Hausa_xlsr_ha_4.2.0_3.0_1664192959924.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Hausa_xlsr_ha_4.2.0_3.0_1664192959924.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Hausa_xlsr", "ha")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Hausa_xlsr", "ha") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Hausa_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ha| |Size:|1.2 GB| --- layout: model title: Legal No waiver Clause Binary Classifier author: John Snow Labs name: legclf_no_waiver_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-waiver` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-waiver` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_waiver_clause_en_1.0.0_3.2_1660122721126.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_waiver_clause_en_1.0.0_3.2_1660122721126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_waiver_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-waiver]| |[other]| |[other]| |[no-waiver]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_waiver_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-waiver 0.95 0.98 0.97 43 other 0.99 0.98 0.99 119 accuracy - - 0.98 162 macro-avg 0.97 0.98 0.98 162 weighted-avg 0.98 0.98 0.98 162 ``` --- layout: model title: Stopwords Remover for Armenian language (105 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, hy, open_source] task: Stop Words Removal language: hy edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_hy_3.4.1_3.0_1646672921601.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_hy_3.4.1_3.0_1646672921601.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------+ |result | +----------------------+ |[ինձանից, ավելի, լավն]| +----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|hy| |Size:|1.8 KB| --- layout: model title: Fast Neural Machine Translation Model from Central Bikol to Spanish author: John Snow Labs name: opus_mt_bcl_es date: 2021-06-01 tags: [open_source, seq2seq, translation, bcl, es, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: bcl target languages: es {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_es_xx_3.1.0_2.4_1622561503404.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_es_xx_3.1.0_2.4_1622561503404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bcl_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English Deberta Embeddings model (from domenicrosati) author: John Snow Labs name: deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt date: 2023-03-12 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, en, tensorflow] task: Embeddings language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deberta-v3-large-dapt-scientific-papers-pubmed-tapt` is a English model originally trained by `domenicrosati`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt_en_4.3.1_3.0_1678658548832.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt_en_4.3.1_3.0_1678658548832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_tapt| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|false| ## References https://huggingface.co/domenicrosati/deberta-v3-large-dapt-scientific-papers-pubmed-tapt --- layout: model title: Javanese DistilBERT Embeddings (Small, Wikipedia) author: John Snow Labs name: distilbert_embeddings_javanese_distilbert_small date: 2022-04-12 tags: [distilbert, embeddings, jv, open_source] task: Embeddings language: jv edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `javanese-distilbert-small` is a Javanese model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_distilbert_small_jv_3.4.2_3.0_1649783759354.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_distilbert_small_jv_3.4.2_3.0_1649783759354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_javanese_distilbert_small| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|248.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-distilbert-small - https://arxiv.org/abs/1910.01108 - https://github.com/sgugger - https://github.com/piegu/fastai-projects/blob/master/finetuning-English-GPT2-any-language-Portuguese-HuggingFace-fastaiv2.ipynb - https://w11wo.github.io/ --- layout: model title: Legal Vacations Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_vacations_bert date: 2023-03-05 tags: [en, legal, classification, clauses, vacations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Vacations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Vacations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_vacations_bert_en_1.0.0_3.0_1678050711001.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_vacations_bert_en_1.0.0_3.0_1678050711001.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_vacations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Vacations]| |[Other]| |[Other]| |[Vacations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_vacations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 1.00 0.98 0.99 56 Vacations 0.97 1.00 0.99 36 accuracy - - 0.99 92 macro-avg 0.99 0.99 0.99 92 weighted-avg 0.99 0.99 0.99 92 ``` --- layout: model title: T5 text-to-text model author: John Snow Labs name: t5_small date: 2020-12-21 task: [Question Answering, Summarization, Translation] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, t5, en] supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The T5 transformer model described in the seminal paper "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer". This model can perform a variety of tasks, such as text summarization, question answering and translation. More details about using the model can be found in the paper (https://arxiv.org/pdf/1910.10683.pdf). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5TRANSFORMER/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_en_2.7.0_2.4_1608554292913.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_en_2.7.0_2.4_1608554292913.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Data Source C4 --- layout: model title: German BertForMaskedLM Base Uncased model (from dbmdz) author: John Snow Labs name: bert_embeddings_base_german_uncased date: 2022-12-02 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-german-uncased` is a German model originally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_german_uncased_de_4.2.4_3.0_1670017726606.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_german_uncased_de_4.2.4_3.0_1670017726606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_german_uncased","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_german_uncased","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_german_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-german-uncased - https://deepset.ai/german-bert - https://deepset.ai/ - https://spacy.io/ - https://github.com/allenai/scibert - https://github.com/stefan-it/fine-tuned-berts-seq - https://github.com/dbmdz/berts/issues/new --- layout: model title: Detect Cellular/Molecular Biology Entities (clinical_large) author: John Snow Labs name: ner_cellular_emb_clinical_large date: 2023-05-24 tags: [ner, en, licensed, clinical, dna, rna, protein, cell_line, cell_type] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for molecular biology related terms. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `DNA`, `Cell_type`, `Cell_line`, `RNA`, `Protein` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CELLULAR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CELLULAR.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cellular_emb_clinical_large_en_4.4.2_3.0_1684920548062.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cellular_emb_clinical_large_en_4.4.2_3.0_1684920548062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_cellular_emb_clinical_large", "en", "clinical/models")\ .setInputCols(["sentence", "token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(['sentence', 'token', 'ner'])\ .setOutputCol('ner_chunk') pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_df = spark.createDataFrame([["""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive."""]]).toDF("text") result = pipeline.fit(sample_df).transform(sample_df) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_cellular_emb_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter)) val sample_data = Seq("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""").toDS.toDF("text") val result = pipeline.fit(sample_data).transform(sample_data) ```

## Results ```bash +-------------------------------------------+-----+---+---------+ |chunk |begin|end|ner_label| +-------------------------------------------+-----+---+---------+ |intracellular signaling proteins |27 |58 |protein | |human T-cell leukemia virus type 1 promoter|130 |172|DNA | |Tax |186 |188|protein | |Tax-responsive element 1 |193 |216|DNA | |cyclic AMP-responsive members |237 |265|protein | |CREB/ATF family |274 |288|protein | |transcription factors |293 |313|protein | |Tax |389 |391|protein | |Tax-responsive element 1 |431 |454|DNA | |TRE-1 |457 |461|DNA | |lacZ gene |582 |590|DNA | |CYC1 promoter |617 |629|DNA | |TRE-1 |663 |667|DNA | |cyclic AMP response element-binding protein|695 |737|protein | |CREB |740 |743|protein | |CREB |749 |752|protein | |GAL4 activation domain |767 |788|protein | |GAD |791 |793|protein | |reporter gene |848 |860|DNA | |Tax |863 |865|protein | +-------------------------------------------+-----+---+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cellular_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.8 MB| ## References Trained on the JNLPBA corpus containing more than 2.404 publication abstracts. (http://www.geniaproject.org/) ## Benchmarking ```bash label precision recall f1-score support cell_type 0.89 0.79 0.84 4912 protein 0.80 0.90 0.84 9841 cell_line 0.66 0.75 0.70 1489 DNA 0.78 0.87 0.82 2845 RNA 0.79 0.81 0.80 305 micro-avg 0.80 0.85 0.83 19392 macro-avg 0.78 0.82 0.80 19392 weighted-avg 0.81 0.85 0.83 19392 ``` --- layout: model title: BERT multilingual base model (cased) author: John Snow Labs name: bert_base_multilingual_cased date: 2021-05-20 tags: [xx, multilingual, embeddings, bert, open_source] supported: true task: Embeddings language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained model on the top 104 languages with the largest Wikipedia using a masked language modeling (MLM) objective. It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in [this repository](https://github.com/google-research/bert). This model is case sensitive: it makes a difference between english and English. BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it was pretrained on the raw texts only, with no humans labeling them in any way (which is why it can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it was pretrained with two objectives: - Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then runs the entire masked sentence through the model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the sentence. - Next sentence prediction (NSP): the models concatenate two masked sentences as inputs during pretraining. Sometimes they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to predict if the two sentences were following each other or not. This way, the model learns an inner representation of the English language that can then be used to extract features useful for downstream tasks: if you have a dataset of labeled sentences, for instance, you can train a standard classifier using the features produced by the BERT model as inputs. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_multilingual_cased_xx_3.1.0_2.4_1621519556711.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_multilingual_cased_xx_3.1.0_2.4_1621519556711.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_multilingual_cased", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_multilingual_cased", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("xx.embed.bert_base_multilingual_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_multilingual_cased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|xx| |Case sensitive:|true| ## Data Source [https://huggingface.co/bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) --- layout: model title: Extract Oncology Tests author: John Snow Labs name: ner_oncology_test date: 2022-11-24 tags: [licensed, clinical, oncology, en, ner, test] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of tests from oncology texts, including pathology tests and imaging tests. Definitions of Predicted Entities: - `Biomarker`: Biological molecules that indicate the presence or absence of cancer, or the type of cancer. Oncogenes are excluded from this category. - `Biomarker_Result`: Terms or values that are identified as the result of a biomarkers. - `Imaging_Test`: Imaging tests mentioned in texts, such as "chest CT scan". - `Oncogene`: Mentions of genes that are implicated in the etiology of cancer. - `Pathology_Test`: Mentions of biopsies or tests that use tissue samples. ## Predicted Entities `Biomarker`, `Biomarker_Result`, `Imaging_Test`, `Oncogene`, `Pathology_Test` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_test_en_4.2.2_3.0_1669307746859.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_test_en_4.2.2_3.0_1669307746859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------------------------|:---------------| | biopsy | Pathology_Test | | ultrasound guided thick-needle | Pathology_Test | | chest computed tomography | Imaging_Test | | CT | Imaging_Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_test| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.2 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Imaging_Test 2020 229 184 2204 0.90 0.92 0.91 Biomarker_Result 1177 186 268 1445 0.86 0.81 0.84 Pathology_Test 888 276 162 1050 0.76 0.85 0.80 Biomarker 1287 254 228 1515 0.84 0.85 0.84 Oncogene 365 89 84 449 0.80 0.81 0.81 macro_avg 5737 1034 926 6663 0.83 0.85 0.84 micro_avg 5737 1034 926 6663 0.85 0.86 0.85 ``` --- layout: model title: English BertForQuestionAnswering model (from deepset) author: John Snow Labs name: bert_base_cased_qa_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-squad2` is a English model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_qa_squad2_en_4.0.0_3.0_1654193845988.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_qa_squad2_en_4.0.0_3.0_1654193845988.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_base_cased_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_base_cased_qa_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.base_cased.by_deepset").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_cased_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/bert-base-cased-squad2 --- layout: model title: Legal Specific performance Clause Binary Classifier author: John Snow Labs name: legclf_specific_performance_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `specific-performance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `specific-performance` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_specific_performance_clause_en_1.0.0_3.2_1660123020947.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_specific_performance_clause_en_1.0.0_3.2_1660123020947.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_specific_performance_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[specific-performance]| |[other]| |[other]| |[specific-performance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_specific_performance_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.98 0.99 94 specific-performance 0.95 1.00 0.97 36 accuracy - - 0.98 130 macro-avg 0.97 0.99 0.98 130 weighted-avg 0.99 0.98 0.98 130 ``` --- layout: model title: Legal Counterparts Clause Binary Classifier author: John Snow Labs name: legclf_counterparts_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `counterparts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `counterparts` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_counterparts_clause_en_1.0.0_3.2_1660123374262.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_counterparts_clause_en_1.0.0_3.2_1660123374262.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_counterparts_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[counterparts]| |[other]| |[other]| |[counterparts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_counterparts_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support counterparts 1.00 1.00 1.00 38 other 1.00 1.00 1.00 97 accuracy - - 1.00 135 macro-avg 1.00 1.00 1.00 135 weighted-avg 1.00 1.00 1.00 135 ``` --- layout: model title: Relation Extraction between different oncological entity types (unspecific version) author: John Snow Labs name: re_oncology_wip date: 2022-09-27 tags: [licensed, clinical, oncology, en, relation_extraction, temporal, test, biomarker, anatomy] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model identifies relations between dates and other clinical entities, between tumor mentions and their size, between anatomical entities and other clinical entities, and between tests and their results. In contrast to re_oncology_granular, all these relation types are labeled as is_related_to. The different types of relations can be identified considering the pairs of entities that are linked. ## Predicted Entities `is_related_to`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_oncology_wip_en_4.0.0_3.0_1664302122205.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_oncology_wip_en_4.0.0_3.0_1664302122205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use realation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash chunk1 entity1 chunk2 entity2 relation confidence mastectomy Cancer_Surgery two months ago Relative_Date is_related_to 0.9623304 3 cm Tumor_Size mass Tumor_Finding is_related_to 0.7947009 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_oncology_wip| |Type:|re| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Size:|266.3 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash relation recall precision f1 O 0.82 0.88 0.85 is_related_to 0.89 0.83 0.86 macro-avg 0.86 0.86 0.86 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Austro-Asiatic languages author: John Snow Labs name: opus_mt_en_aav date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, aav, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `aav` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_aav_xx_2.7.0_2.4_1609169278246.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_aav_xx_2.7.0_2.4_1609169278246.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_aav| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from CNT-UPenn) author: John Snow Labs name: roberta_qa_for_seizurefrequency date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RoBERTa_for_seizureFrequency_QA` is a English model originally trained by `CNT-UPenn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_for_seizurefrequency_en_4.3.0_3.0_1674208667059.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_for_seizurefrequency_en_4.3.0_3.0_1674208667059.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_for_seizurefrequency","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_for_seizurefrequency","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_for_seizurefrequency| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/CNT-UPenn/RoBERTa_for_seizureFrequency_QA - https://doi.org/10.1093/jamia/ocac018 --- layout: model title: Pipeline to Detect Living Species(w2v_cc_300d) author: John Snow Labs name: ner_living_species_pipeline date: 2023-03-13 tags: [pt, ner, clinical, licensed] task: Named Entity Recognition language: pt edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species](https://nlp.johnsnowlabs.com/2022/06/22/ner_living_species_pt_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_pt_4.3.0_3.2_1678708110628.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_pt_4.3.0_3.2_1678708110628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_pipeline", "pt", "clinical/models") text = '''Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_pipeline", "pt", "clinical/models") val text = "Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:------------|-------------:| | 0 | rapariga | 4 | 11 | HUMAN | 0.9991 | | 1 | pessoal | 41 | 47 | HUMAN | 0.9765 | | 2 | paciente | 182 | 189 | HUMAN | 1 | | 3 | gato | 368 | 371 | SPECIES | 0.9847 | | 4 | veterinário | 413 | 423 | HUMAN | 0.91 | | 5 | Trichophyton rubrum | 632 | 650 | SPECIES | 0.9996 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_vasilis TFWav2Vec2ForCTC from vasilis author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_finnish_by_vasilis date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_vasilis` is a Finnish model originally trained by vasilis. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_fi_4.2.0_3.0_1664024039687.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_fi_4.2.0_3.0_1664024039687.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_vasilis", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_vasilis", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_finnish_by_vasilis| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: English T5ForConditionalGeneration Cased model (from yirmibesogluz) author: John Snow Labs name: t5_t2t_ner_ade_balanced date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t2t-ner-ade-balanced` is a English model originally trained by `yirmibesogluz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_t2t_ner_ade_balanced_en_4.3.0_3.0_1675107775759.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_t2t_ner_ade_balanced_en_4.3.0_3.0_1675107775759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_t2t_ner_ade_balanced","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_t2t_ner_ade_balanced","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_t2t_ner_ade_balanced| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|924.7 MB| ## References - https://huggingface.co/yirmibesogluz/t2t-ner-ade-balanced - https://github.com/gokceuludogan/boun-tabi-smm4h22 --- layout: model title: Legal Natural Environment Document Classifier (EURLEX) author: John Snow Labs name: legclf_natural_environment_bert date: 2023-03-06 tags: [en, legal, classification, clauses, natural_environment, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_natural_environment_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Natural_Environment or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Natural_Environment`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_natural_environment_bert_en_1.0.0_3.0_1678111551323.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_natural_environment_bert_en_1.0.0_3.0_1678111551323.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_natural_environment_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Natural_Environment]| |[Other]| |[Other]| |[Natural_Environment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_natural_environment_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Natural_Environment 0.89 0.87 0.88 45 Other 0.88 0.90 0.89 50 accuracy - - 0.88 95 macro-avg 0.88 0.88 0.88 95 weighted-avg 0.88 0.88 0.88 95 ``` --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from jonfrank) author: John Snow Labs name: xlmroberta_ner_jonfrank_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `jonfrank`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jonfrank_base_finetuned_panx_de_4.1.0_3.0_1660434803462.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jonfrank_base_finetuned_panx_de_4.1.0_3.0_1660434803462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jonfrank_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jonfrank_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jonfrank_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jonfrank/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Italian T5ForConditionalGeneration Base Cased model (from aiknowyou) author: John Snow Labs name: t5_mt5_base_it_paraphraser date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mt5-base-it-paraphraser` is a Italian model originally trained by `aiknowyou`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_mt5_base_it_paraphraser_it_4.3.0_3.0_1675105866508.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_mt5_base_it_paraphraser_it_4.3.0_3.0_1675105866508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_mt5_base_it_paraphraser","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_mt5_base_it_paraphraser","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_mt5_base_it_paraphraser| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|969.5 MB| ## References - https://huggingface.co/aiknowyou/mt5-base-it-paraphraser - https://arxiv.org/abs/2010.11934 - https://colab.research.google.com/drive/1DGeF190gJ3DjRFQiwhFuZalp427iqJNQ - https://gist.github.com/avidale/44cd35bfcdaf8bedf51d97c468cc8001 - http://creativecommons.org/licenses/by-nc-sa/4.0/ - http://creativecommons.org/licenses/by-nc-sa/4.0/ --- layout: model title: Detect tumor morphology in Spanish texts author: John Snow Labs name: cantemist_scielowiki date: 2021-07-23 tags: [ner, licensed, oncology, es] task: Named Entity Recognition language: es edition: Spark NLP for Healthcare 3.1.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect tumor morphology entities in Spanish text. ## Predicted Entities `MORFOLOGIA_NEOPLASIA`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_TUMOR_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/cantemist_scielowiki_es_3.1.2_3.0_1627080305994.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/cantemist_scielowiki_es_3.1.2_3.0_1627080305994.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") embedings_stage = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d", "es", "clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") clinical_ner = MedicalNerModel.pretrained("cantemist_scielowiki", "es", "clinical/models")\ .setInputCols(["sentence", "token", "word_embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(['document', 'token', 'ner']) \ .setOutputCol('ner_chunk') pipeline = Pipeline(stages=[ document_assembler, sentence, tokenizer, embedings_stage, clinical_ner, ner_converter ]) data = spark.createDataFrame([["""Anamnesis Paciente de 37 años de edad sin antecedentes patológicos ni quirúrgicos de interés. En diciembre de 2012 consultó al Servicio de Urgencias por un cuadro de cefalea aguda e hipostesia del hemicuerpo izquierdo de 15 días de evolución refractario a tratamiento. Exploración neurológica sin focalidad; fondo de ojo: papiledema unilateral. Se solicitaron una TC del SNC, que objetiva una LOE frontal derecha con afectación aparente del cuerpo calloso, y una RM del SNC, que muestra un extenso proceso expansivo intraparenquimatoso frontal derecho que infiltra la rodilla del cuerpo calloso, mal delimitada y sin componente necrótico. Tras la administración de contraste se apreciaban diferentes realces parcheados en la lesión, pero sin definirse una cápsula con aumento del flujo sanguíneo en la lesión, características compatibles con linfoma o astrocitoma anaplásico . El 3 de enero de 2013 se efectúa biopsia intraoperatoria, con diagnóstico histológico de astrocitoma anaplásico GIII"""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embedings_stage = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d", "es", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") val clinical_ner = MedicalNerModel.pretrained("cantemist_scielowiki", "es", "clinical/models") .setInputCols(Array("sentence", "token", "word_embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence, tokenizer, embedings_stage, clinical_ner, ner_converter)) val data = Seq("""Anamnesis Paciente de 37 años de edad sin antecedentes patológicos ni quirúrgicos de interés. En diciembre de 2012 consultó al Servicio de Urgencias por un cuadro de cefalea aguda e hipostesia del hemicuerpo izquierdo de 15 días de evolución refractario a tratamiento. Exploración neurológica sin focalidad; fondo de ojo: papiledema unilateral. Se solicitaron una TC del SNC, que objetiva una LOE frontal derecha con afectación aparente del cuerpo calloso, y una RM del SNC, que muestra un extenso proceso expansivo intraparenquimatoso frontal derecho que infiltra la rodilla del cuerpo calloso, mal delimitada y sin componente necrótico. Tras la administración de contraste se apreciaban diferentes realces parcheados en la lesión, pero sin definirse una cápsula con aumento del flujo sanguíneo en la lesión, características compatibles con linfoma o astrocitoma anaplásico . El 3 de enero de 2013 se efectúa biopsia intraoperatoria, con diagnóstico histológico de astrocitoma anaplásico GIII""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +---------------------+----------------------+ | token | prediction | +---------------------+----------------------+ | Anamnesis | O | | Paciente | O | | de | O | | 37 | O | | años | O | | de | O | | edad | O | | sin | O | | antecedentes | O | | patológicos | O | | ni | O | | quirúrgicos | O | | de | O | | interés | O | | . | O | | En | O | | diciembre | O | | de | O | | 2012 | O | | consultó | O | | al | O | | Servicio | O | | de | O | | Urgencias | O | | por | O | | un | O | | cuadro | O | | de | O | | cefalea | O | | aguda | O | | e | O | | hipostesia | O | | del | O | | hemicuerpo | O | | izquierdo | O | | de | O | | 15 | O | | días | O | | de | O | | evolución | O | | refractario | O | | a | O | | tratamiento | O | | . | O | | Exploración | O | | neurológica | O | | sin | O | | focalidad | O | | ; | O | | fondo | O | | de | O | | ojo | O | | : | O | | papiledema | O | | unilateral | O | | . | O | | Se | O | | solicitaron | O | | una | O | | TC | O | | del | O | | SNC | B-MORFOLOGIA_NEOP... | | , | O | | que | O | | objetiva | O | | una | O | | LOE | O | | frontal | O | | derecha | O | | con | O | | afectación | B-MORFOLOGIA_NEOP... | | aparente | I-MORFOLOGIA_NEOP... | | del | I-MORFOLOGIA_NEOP... | | cuerpo | I-MORFOLOGIA_NEOP... | | calloso | I-MORFOLOGIA_NEOP... | | , | O | | y | O | | una | O | | RM | B-MORFOLOGIA_NEOP... | | del | I-MORFOLOGIA_NEOP... | | SNC | I-MORFOLOGIA_NEOP... | | , | O | | que | O | | muestra | O | | un | O | | extenso | O | | proceso | B-MORFOLOGIA_NEOP... | | expansivo | I-MORFOLOGIA_NEOP... | | intraparenquimatoso | I-MORFOLOGIA_NEOP... | | frontal | I-MORFOLOGIA_NEOP... | | derecho | I-MORFOLOGIA_NEOP... | | que | I-MORFOLOGIA_NEOP... | | infiltra | I-MORFOLOGIA_NEOP... | | la | I-MORFOLOGIA_NEOP... | | rodilla | I-MORFOLOGIA_NEOP... | | del | I-MORFOLOGIA_NEOP... | | cuerpo | I-MORFOLOGIA_NEOP... | | calloso | I-MORFOLOGIA_NEOP... | | , | O | | mal | O | | delimitada | O | | y | O | | sin | O | | componente | O | | necrótico | O | | . | O | | Tras | O | | la | O | | administración | O | | de | O | | contraste | O | | se | O | | apreciaban | O | | diferentes | O | | realces | O | | parcheados | O | | en | O | | la | O | | lesión | O | | , | O | | pero | O | | sin | O | | definirse | O | | una | O | | cápsula | O | | con | O | | aumento | O | | del | O | | flujo | O | | sanguíneo | O | | en | O | | la | O | | lesión | O | | , | O | | características | O | | compatibles | O | | con | O | | linfoma | O | | o | O | | astrocitoma | B-MORFOLOGIA_NEOP... | | anaplásico | I-MORFOLOGIA_NEOP... | | . | O | | El | O | | 3 | O | | de | O | | enero | O | | de | O | | 2013 | O | | se | O | | efectúa | O | | biopsia | O | | intraoperatoria | O | | , | O | | con | O | | diagnóstico | O | | histológico | O | | de | O | | astrocitoma | B-MORFOLOGIA_NEOP... | | anaplásico | I-MORFOLOGIA_NEOP... | | GIII | I-MORFOLOGIA_NEOP... | +---------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|cantemist_scielowiki| |Compatibility:|Spark NLP for Healthcare 3.1.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Dependencies:|embeddings_scielowiki_300d| ## Data Source The model was trained with the [CANTEMIST](https://temu.bsc.es/cantemist/) data set: > CANTEMIST is an annotated data set for oncology analysis in the Spanish language containing 1301 oncological clinical case reports with a total of 63,016 sentences and 1093,501 tokens. All documents of the corpus have been manually annotated by clinical experts with mentions of tumor morphology (in Spanish, “morfología de neoplasia”). There are 16,030 tumor morphology mentions mapped to an eCIE-O code (850 unique codes) References: 1. P. Ruas, A. Neves, V. D. Andrade, F. M. Couto, Lasigebiotm at cantemist: Named entity recognition and normalization of tumour morphology entities and clinical coding of Spanish health-related documents, in: Proceedings of the Iberian Languages Evaluation Forum (IberLEF 2020), CEUR Workshop Proceedings, 2020 2. Antonio Miranda-Escalada, Eulàlia Farré-Maduell, Martin Krallinger. Named Entity Recognition, Concept Normalization and Clinical Coding: Overview of the Cantemist Track for Cancer Text Mining in Spanish, Corpus, Guidelines, Methods and Results. Proceedings of the Iberian Languages Evaluation Forum (IberLEF 2020), CEUR Workshop Proceedings. 303-323 (2020). ## Benchmarking ```bash label precision recall f1-score support B-MORFOLOGIA_NEOPLASIA 0.94 0.73 0.83 2474 I-MORFOLOGIA_NEOPLASIA 0.81 0.74 0.77 3169 O 0.99 1.00 1.00 283006 accuracy - - 0.99 288649 macro-avg 0.92 0.82 0.87 288649 weighted-avg 0.99 0.99 0.99 288649 ``` --- layout: model title: Yue Chinese asr_wav2vec2_large_xlsr_cantonese_by_ctl TFWav2Vec2ForCTC from ctl author: John Snow Labs name: asr_wav2vec2_large_xlsr_cantonese_by_ctl date: 2022-09-24 tags: [wav2vec2, yue, audio, open_source, asr] task: Automatic Speech Recognition language: yue edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_cantonese_by_ctl` is a Yue Chinese model originally trained by ctl. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_cantonese_by_ctl_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_cantonese_by_ctl_yue_4.2.0_3.0_1664039699316.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_cantonese_by_ctl_yue_4.2.0_3.0_1664039699316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_cantonese_by_ctl", "yue")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_cantonese_by_ctl", "yue") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_cantonese_by_ctl| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|yue| |Size:|1.2 GB| --- layout: model title: Google's Tapas Table Understanding (Small, SQA) author: John Snow Labs name: table_qa_tapas_small_finetuned_sqa date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Small Has aggregation operations?: False ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_small_finetuned_sqa_en_4.2.0_3.0_1664530724535.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_small_finetuned_sqa_en_4.2.0_3.0_1664530724535.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_small_finetuned_sqa","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_gv_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2_en_4.3.0_3.0_1674214328836.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2_en_4.3.0_3.0_1674214328836.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_16_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|416.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-2 --- layout: model title: English DistilBertForQuestionAnswering Cased model (from rowan1224) author: John Snow Labs name: distilbert_qa_squad_slp date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-squad-slp` is a English model originally trained by `rowan1224`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_slp_en_4.3.0_3.0_1672774348522.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_slp_en_4.3.0_3.0_1672774348522.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_slp","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_slp","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_slp| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rowan1224/distilbert-squad-slp --- layout: model title: Legal Subordination Clause Binary Classifier author: John Snow Labs name: legclf_subordination_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `subordination` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `subordination` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subordination_clause_en_1.0.0_3.2_1660124022192.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subordination_clause_en_1.0.0_3.2_1660124022192.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_subordination_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[subordination]| |[other]| |[other]| |[subordination]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_subordination_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.98 0.96 49 subordination 0.96 0.89 0.93 28 accuracy - - 0.95 77 macro-avg 0.95 0.94 0.94 77 weighted-avg 0.95 0.95 0.95 77 ``` --- layout: model title: Detect Living Species (bert_embeddings_bert_base_italian_xxl_cased) author: John Snow Labs name: ner_living_species_bert date: 2022-06-23 tags: [it, ner, clinical, licensed, bert] task: Named Entity Recognition language: it edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Italian which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `bert_embeddings_bert_base_italian_xxl_cased` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_it_3.5.3_3.0_1655972219820.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_it_3.5.3_3.0_1655972219820.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_bert_base_italian_xxl_cased", "it")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "it", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_bert_base_italian_xxl_cased", "it") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "it", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("it.med_ner.living_species.bert").predict("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""") ```

## Results ```bash +----------------+-------+ |ner_chunk |label | +----------------+-------+ |donna |HUMAN | |personale |HUMAN | |madre |HUMAN | |fratello |HUMAN | |sorella |HUMAN | |virus epatotropi|SPECIES| |HBV |SPECIES| |HCV |SPECIES| |HIV |SPECIES| |paziente |HUMAN | +----------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|it| |Size:|16.4 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.88 0.95 0.91 2772 B-SPECIES 0.76 0.89 0.82 2860 I-HUMAN 0.70 0.59 0.64 101 I-SPECIES 0.70 0.81 0.75 1036 micro-avg 0.80 0.90 0.85 6769 macro-avg 0.76 0.81 0.78 6769 weighted-avg 0.80 0.90 0.85 6769 ``` --- layout: model title: English asr_wav2vec_finetuned_on_cryptocurrency TFWav2Vec2ForCTC from distractedm1nd author: John Snow Labs name: asr_wav2vec_finetuned_on_cryptocurrency date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec_finetuned_on_cryptocurrency` is a English model originally trained by distractedm1nd. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec_finetuned_on_cryptocurrency_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec_finetuned_on_cryptocurrency_en_4.2.0_3.0_1664024959023.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec_finetuned_on_cryptocurrency_en_4.2.0_3.0_1664024959023.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec_finetuned_on_cryptocurrency", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec_finetuned_on_cryptocurrency", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec_finetuned_on_cryptocurrency| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: RE Pipeline between Tests, Results, and Dates author: John Snow Labs name: re_test_result_date_pipeline date: 2023-06-13 tags: [licensed, clinical, relation_extraction, tests, results, dates, en] task: Relation Extraction language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_test_result_date](https://nlp.johnsnowlabs.com/2021/02/24/re_test_result_date_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_test_result_date_pipeline_en_4.4.4_3.2_1686665254277.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_test_result_date_pipeline_en_4.4.4_3.2_1686665254277.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_test_result_date_pipeline", "en", "clinical/models") pipeline.fullAnnotate("He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_test_result_date_pipeline", "en", "clinical/models") pipeline.fullAnnotate("He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.date_test_result.pipeline").predict("""He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%""") ```

## Results ```bash Results | index | relations | entity1 | chunk1 | entity2 | chunk2 | |-------|--------------|--------------|---------------------|--------------|---------| | 0 | O | TEST | chest X-ray | MEASUREMENTS | 93% | | 1 | O | TEST | CT scan | MEASUREMENTS | 93% | | 2 | is_result_of | TEST | SpO2 | MEASUREMENTS | 93% | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_test_result_date_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: English BertForQuestionAnswering model (from aymanm419) author: John Snow Labs name: bert_qa_araSpeedest date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `araSpeedest` is a English model orginally trained by `aymanm419`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_araSpeedest_en_4.0.0_3.0_1654179104397.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_araSpeedest_en_4.0.0_3.0_1654179104397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_araSpeedest","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_araSpeedest","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_aymanm419").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_araSpeedest| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|505.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aymanm419/araSpeedest --- layout: model title: Legal No material adverse change Clause Binary Classifier author: John Snow Labs name: legclf_no_material_adverse_change_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-material-adverse-change` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-material-adverse-change` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_material_adverse_change_clause_en_1.0.0_3.2_1660122691534.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_material_adverse_change_clause_en_1.0.0_3.2_1660122691534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_material_adverse_change_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-material-adverse-change]| |[other]| |[other]| |[no-material-adverse-change]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_material_adverse_change_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-material-adverse-change 0.95 0.97 0.96 37 other 0.99 0.98 0.99 103 accuracy - - 0.98 140 macro-avg 0.97 0.98 0.97 140 weighted-avg 0.98 0.98 0.98 140 ``` --- layout: model title: Detect Anatomical Regions (MedicalBertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_anatomy date: 2022-01-06 tags: [anatomy, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for anatomy terms. This model is trained with the BertForTokenClassification method from the transformers library and imported into Spark NLP. ## Predicted Entities `Anatomical_system`, `Cell`, `Cellular_component`, `Developing_anatomical_structure`, `Immaterial_anatomical_entity`, `Multi-tissue_structure`, `Organ`, `Organism_subdivision`, `Organism_substance`, `Pathological_formation`, `Tissue` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_en_3.3.4_2.4_1641454747169.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_en_3.3.4_2.4_1641454747169.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_anatomy", "en", "clinical/models")\ .setInputCols("token", "document")\ .setOutputCol("ner")\ .setCaseSensitive(True) ner_converter = NerConverter() \ .setInputCols(["document","token","ner"]) \ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier, ner_converter]) data = spark.createDataFrame([["""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_anatomy", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("ner") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("document","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier, ner_converter)) val data = Seq("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_anatomy").predict("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""") ```

## Results ```bash +-------------------+----------------------+ |chunk |ner_label | +-------------------+----------------------+ |great toe |Multi-tissue_structure| |skin |Organ | |conjunctivae |Multi-tissue_structure| |Extraocular muscles|Multi-tissue_structure| |Nares |Multi-tissue_structure| |turbinates |Multi-tissue_structure| |Oropharynx |Multi-tissue_structure| |Mucous membranes |Tissue | |Neck |Organism_subdivision | |bowel |Organ | |great toe |Multi-tissue_structure| |skin |Organ | |toenails |Organism_subdivision | |foot |Organism_subdivision | |great toe |Multi-tissue_structure| |toenails |Organism_subdivision | +-------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_anatomy| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.4 MB| |Case sensitive:|true| |Max sentense length:|512| ## Data Source Trained on the Anatomical Entity Mention (AnEM) corpus with 'embeddings_clinical'. http://www.nactem.ac.uk/anatomy/ ## Benchmarking ```bash label precision recall f1-score support B-Anatomical_system 1.00 0.50 0.67 4 B-Cell 0.89 0.96 0.92 74 B-Cellular_component 0.97 0.81 0.88 36 B-Developing_anatomical_structure 1.00 0.50 0.67 6 B-Immaterial_anatomical_entity 0.60 0.75 0.67 4 B-Multi-tissue_structure 0.75 0.86 0.80 58 B-Organ 0.86 0.88 0.87 48 B-Organism_subdivision 0.62 0.42 0.50 12 B-Organism_substance 0.89 0.81 0.85 31 B-Pathological_formation 0.91 0.91 0.91 32 B-Tissue 0.94 0.76 0.84 21 I-Anatomical_system 1.00 1.00 1.00 1 I-Cell 1.00 0.84 0.91 62 I-Cellular_component 0.92 0.85 0.88 13 I-Developing_anatomical_structure 1.00 1.00 1.00 1 I-Immaterial_anatomical_entity 1.00 1.00 1.00 1 I-Multi-tissue_structure 1.00 0.77 0.87 26 I-Organ 0.80 0.80 0.80 5 I-Organism_substance 1.00 0.71 0.83 7 I-Pathological_formation 1.00 0.94 0.97 16 I-Tissue 0.93 0.93 0.93 15 accuracy - - 0.84 473 macro-avg 0.87 0.77 0.83 473 weighted-avg 0.90 0.84 0.86 473 ``` --- layout: model title: Part of Speech for Latin author: John Snow Labs name: pos_ud_llct date: 2020-07-29 23:34:00 +0800 task: Part of Speech Tagging language: la edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [pos, la] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_llct_la_2.5.5_2.4_1596054191115.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_llct_la_2.5.5_2.4_1596054191115.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_llct", "la") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_llct", "la") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene."""] pos_df = nlu.load('la.pos').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='PROPN', metadata={'word': 'Alius'}), Row(annotatorType='pos', begin=6, end=8, result='AUX', metadata={'word': 'est'}), Row(annotatorType='pos', begin=10, end=13, result='VERB', metadata={'word': 'esse'}), Row(annotatorType='pos', begin=15, end=19, result='VERB', metadata={'word': 'regem'}), Row(annotatorType='pos', begin=21, end=29, result='PROPN', metadata={'word': 'Aquilonis'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_llct| |Type:|pos| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|la| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Japanese T5ForConditionalGeneration Cased model (from astremo) author: John Snow Labs name: t5_friendly date: 2023-01-30 tags: [ja, open_source, t5, tensorflow] task: Text Generation language: ja edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `friendly_JA` is a Japanese model originally trained by `astremo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_friendly_ja_4.3.0_3.0_1675102435483.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_friendly_ja_4.3.0_3.0_1675102435483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_friendly","ja") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_friendly","ja") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_friendly| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ja| |Size:|923.1 MB| ## References - https://huggingface.co/astremo/friendly_JA - http://creativecommons.org/licenses/by/4.0/ - http://creativecommons.org/licenses/by/4.0/ - http://creativecommons.org/licenses/by/4.0/ --- layout: model title: English RobertaForQuestionAnswering Cased model (from skandaonsolve) author: John Snow Labs name: roberta_qa_finetuned_timeentities2_ttsp75 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-timeentities2_ttsp75` is a English model originally trained by `skandaonsolve`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities2_ttsp75_en_4.3.0_3.0_1674220728523.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities2_ttsp75_en_4.3.0_3.0_1674220728523.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities2_ttsp75","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities2_ttsp75","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_timeentities2_ttsp75| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-timeentities2_ttsp75 --- layout: model title: Fast Neural Machine Translation Model from Hausa to English author: John Snow Labs name: opus_mt_ha_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ha, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ha` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ha_en_xx_2.7.0_2.4_1609168807678.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ha_en_xx_2.7.0_2.4_1609168807678.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ha_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Financial Zero-shot NER author: John Snow Labs name: finner_roberta_zeroshot date: 2022-09-02 tags: [en, finance, ner, zero, shot, zeroshot, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true recommended: true annotator: ZeroShotNER article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to carry out a Zero-Shot Named Entity Recognition (NER) approach, detecting any kind of entities with no training dataset, just tje pretrained RoBERTa embeddings (included in the model) and some examples. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_ZEROSHOT/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_roberta_zeroshot_en_1.0.0_3.2_1662113599526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_roberta_zeroshot_en_1.0.0_3.2_1662113599526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sparktokenizer = nlp.Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") zero_shot_ner = finance.ZeroShotNerModel.pretrained("finner_roberta_zeroshot", "en", "finance/models")\ .setInputCols(["document", "token"])\ .setOutputCol("zero_shot_ner")\ .setEntityDefinitions( { "DATE": ['When was the company acquisition?', 'When was the company purchase agreement?'], "ORG": ["Which company was acquired?"], "PRODUCT": ["Which product?"], "PROFIT_INCREASE": ["How much has the gross profit increased?"], "REVENUES_DECLINED": ["How much has the revenues declined?"], "OPERATING_LOSS_2020": ["Which was the operating loss in 2020"], "OPERATING_LOSS_2019": ["Which was the operating loss in 2019"] }) nerconverter = nlp.NerConverter()\ .setInputCols(["document", "token", "zero_shot_ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ documentAssembler, sparktokenizer, zero_shot_ner, nerconverter, ] ) sample_text = ["In March 2012, as part of a longer-term strategy, the Company acquired Vertro, Inc., which owned and operated the ALOT product portfolio.", "In February 2017, the Company entered into an asset purchase agreement with NetSeer, Inc.", "While our gross profit margin increased to 81.4% in 2020 from 63.1% in 2019, our revenues declined approximately 27% in 2020 as compared to 2019." "We reported an operating loss of approximately $8,048,581 million in 2020 as compared to an operating loss of approximately $7,738,193 million in 2019."] p_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) res = p_model.transform(spark.createDataFrame(sample_text, StringType()).toDF("text")) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.begin, res.ner_chunk.end, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['0']").alias("chunk"), F.expr("cols['3']['entity']").alias("ner_label"))\ .filter("ner_label!='O'")\ .show(truncate=False) ```

## Results ```bash +------------------+-------------------+ |chunk |ner_label | +------------------+-------------------+ |March 2012 |DATE | |Vertro |ORG | |ALOT |PRODUCT | |February 2017 |DATE | |NetSeer |ORG | |81.4% |PROFIT_INCREASE | |27% |REVENUES_DECLINED | |$8,048,581 million|OPERATING_LOSS_2020| |$7,738,193 million|OPERATING_LOSS_2019| +------------------+-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_roberta_zeroshot| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References Financial Roberta Embeddings --- layout: model title: Malay T5ForConditionalGeneration Base Cased model (from mesolitica) author: John Snow Labs name: t5_base_bahasa_cased date: 2023-01-30 tags: [ms, open_source, t5, tensorflow] task: Text Generation language: ms edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-bahasa-cased` is a Malay model originally trained by `mesolitica`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_bahasa_cased_ms_4.3.0_3.0_1675108290125.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_bahasa_cased_ms_4.3.0_3.0_1675108290125.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_bahasa_cased","ms") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_bahasa_cased","ms") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_bahasa_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ms| |Size:|473.3 MB| ## References - https://huggingface.co/mesolitica/t5-base-bahasa-cased - https://github.com/huseinzol05/malaya/tree/master/pretrained-model/t5/prepare - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5 --- layout: model title: English RobertaForQuestionAnswering Cased model (from LucasS) author: John Snow Labs name: roberta_qa_robertabaseabsa date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robertaBaseABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_robertabaseabsa_en_4.3.0_3.0_1674222849343.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_robertabaseabsa_en_4.3.0_3.0_1674222849343.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertabaseabsa","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertabaseabsa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_robertabaseabsa| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|437.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/LucasS/robertaBaseABSA --- layout: model title: Arabic BertForMaskedLM Base Cased model (from CAMeL-Lab) author: John Snow Labs name: bert_embeddings_base_arabic_camel_msa date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabic-camelbert-msa` is a Arabic model originally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_ar_4.2.4_3.0_1670016029582.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_ar_4.2.4_3.0_1670016029582.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabic_camel_msa| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: French RoBERTa Embeddings (from benjamin) author: John Snow Labs name: roberta_embeddings_roberta_base_wechsel_french date: 2022-04-14 tags: [roberta, embeddings, fr, open_source] task: Embeddings language: fr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-base-wechsel-french` is a French model orginally trained by `benjamin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_base_wechsel_french_fr_3.4.2_3.0_1649947929675.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_base_wechsel_french_fr_3.4.2_3.0_1649947929675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_base_wechsel_french| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fr| |Size:|468.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/benjamin/roberta-base-wechsel-french - https://github.com/CPJKU/wechsel - https://arxiv.org/abs/2112.06598 --- layout: model title: Stop Words Cleaner for Basque author: John Snow Labs name: stopwords_eu date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: eu edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, eu] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_eu_eu_2.5.4_2.4_1594742441951.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_eu_eu_2.5.4_2.4_1594742441951.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_eu", "eu") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_eu", "eu") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow."""] stopword_df = nlu.load('eu.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=10, result='Iparraldeko', metadata={'sentence': '0'}), Row(annotatorType='token', begin=12, end=18, result='erregea', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=26, result='izateaz', metadata={'sentence': '0'}), Row(annotatorType='token', begin=28, end=31, result='gain', metadata={'sentence': '0'}), Row(annotatorType='token', begin=32, end=32, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_eu| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|eu| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: ICDO Entity Resolver author: John Snow Labs name: chunkresolve_icdo_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-21 task: Entity Resolution edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD-O Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICDO/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICDO.ipynb#scrollTo=Qdh2BQaLI7tU){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icdo_clinical_en_2.4.5_2.4_1587491354644.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icdo_clinical_en_2.4.5_2.4_1587491354644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... model = ChunkEntityResolverModel.pretrained("chunkresolve_icdo_clinical","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, clinical_ner_model, clinical_ner_chunker, chunk_embeddings, model]) data = ["""DIAGNOSIS: Left breast adenocarcinoma stage T3 N1b M0, stage IIIA. She has been found more recently to have stage IV disease with metastatic deposits and recurrence involving the chest wall and lower left neck lymph nodes. PHYSICAL EXAMINATION NECK: On physical examination palpable lymphadenopathy is present in the left lower neck and supraclavicular area. No other cervical lymphadenopathy or supraclavicular lymphadenopathy is present. RESPIRATORY: Good air entry bilaterally. Examination of the chest wall reveals a small lesion where the chest wall recurrence was resected. No lumps, bumps or evidence of disease involving the right breast is present. ABDOMEN: Normal bowel sounds, no hepatomegaly. No tenderness on deep palpation. She has just started her last cycle of chemotherapy today, and she wishes to visit her daughter in Brooklyn, New York. After this she will return in approximately 3 to 4 weeks and begin her radiotherapy treatment at that time."""] pipeline_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline = LightPipeline(pipeline_model) result = light_pipeline.annotate(data) ``` ```scala ... val model = ChunkEntityResolverModel.pretrained("chunkresolve_icdo_clinical","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, clinical_ner_model, clinical_ner_chunker, chunk_embeddings, model)) val data = Seq("DIAGNOSIS: Left breast adenocarcinoma stage T3 N1b M0, stage IIIA. She has been found more recently to have stage IV disease with metastatic deposits and recurrence involving the chest wall and lower left neck lymph nodes. PHYSICAL EXAMINATION NECK: On physical examination palpable lymphadenopathy is present in the left lower neck and supraclavicular area. No other cervical lymphadenopathy or supraclavicular lymphadenopathy is present. RESPIRATORY: Good air entry bilaterally. Examination of the chest wall reveals a small lesion where the chest wall recurrence was resected. No lumps, bumps or evidence of disease involving the right breast is present. ABDOMEN: Normal bowel sounds, no hepatomegaly. No tenderness on deep palpation. She has just started her last cycle of chemotherapy today, and she wishes to visit her daughter in Brooklyn, New York. After this she will return in approximately 3 to 4 weeks and begin her radiotherapy treatment at that time.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash | | chunk | begin | end | entity | idco_description | icdo_code | |---|----------------------------|-------|-----|--------|---------------------------------------------|-----------| | 0 | Left breast adenocarcinoma | 11 | 36 | Cancer | Intraductal carcinoma, noninfiltrating, NOS | 8500/2 | | 1 | T3 N1b M0 | 44 | 52 | Cancer | Kaposi sarcoma | 9140/3 | ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------| | Name: | chunkresolve_icdo_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.2+ | | License: | Licensed | |Edition:|Official| | |Input labels: | token, chunk_embeddings | |Output labels: | entity | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD-O Histology Behaviour dataset https://apps.who.int/iris/bitstream/handle/10665/96612/9789241548496_eng.pdf --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from FOFer) author: John Snow Labs name: distilbert_qa_fofer_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `FOFer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_fofer_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768487186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_fofer_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768487186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_fofer_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_fofer_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_fofer_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/FOFer/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_10_H_512_A_8_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-10_H-512_A-8_squad2` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_10_H_512_A_8_squad2_en_4.0.0_3.0_1654185195977.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_10_H_512_A_8_squad2_en_4.0.0_3.0_1654185195977.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_10_H_512_A_8_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_10_H_512_A_8_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.uncased_10l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_10_H_512_A_8_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|178.3 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-10_H-512_A-8_squad2 --- layout: model title: Arabic Part of Speech Tagger (from CAMeL-Lab) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_ca_pos_egy date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-ca-pos-egy` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_egy_ar_3.4.2_3.0_1650993368525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_egy_ar_3.4.2_3.0_1650993368525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_ca_pos_egy| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-ca-pos-egy - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: Legal Confidentiality Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_confidentiality_bert date: 2023-03-05 tags: [en, legal, classification, clauses, confidentiality, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Confidentiality` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Confidentiality`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_confidentiality_bert_en_1.0.0_3.0_1678050626282.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_confidentiality_bert_en_1.0.0_3.0_1678050626282.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_confidentiality_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Confidentiality]| |[Other]| |[Other]| |[Confidentiality]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_confidentiality_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Confidentiality 0.94 0.99 0.96 123 Other 0.99 0.95 0.97 150 accuracy - - 0.97 273 macro-avg 0.97 0.97 0.97 273 weighted-avg 0.97 0.97 0.97 273 ``` --- layout: model title: Word Embeddings for Japanese (japanese_cc_300d) author: John Snow Labs name: japanese_cc_300d date: 2021-09-09 tags: [embeddings, open_source, ja] task: Embeddings language: ja edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/japanese_cc_300d_ja_3.2.2_3.0_1631192388744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/japanese_cc_300d_ja_3.2.2_3.0_1631192388744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+ | col| +---------------------------+ | [宮本, [0.1944, 0.4...| | [茂, [-0.079, 0.09...| | [氏, [-0.1053, 0.1...| | [は, [0.0732, -0.0...| | [、, [0.0571, -0.0...| | [日本, [0.1844, 0.0...| | [の, [0.0109, -0.0...| | [任天, [0.0, 0.0, 0...| | [堂, [-0.1972, 0.0...| | [の, [0.0109, -0.0...| | [ゲーム, [0.013, 0.0...| |[プロデューサー, [-0.010...| | [です, [0.0036, -0....| | [。, [0.069, -0.01...| +---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|japanese_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ja| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_fpdm_triplet_ft_new_news date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fpdm_triplet_roberta_FT_new_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_triplet_ft_new_news_en_4.3.0_3.0_1674211184082.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_triplet_ft_new_news_en_4.3.0_3.0_1674211184082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_triplet_ft_new_news","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_triplet_ft_new_news","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fpdm_triplet_ft_new_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|461.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/fpdm_triplet_roberta_FT_new_newsqa --- layout: model title: Legal Indemnifications Clause Binary Classifier author: John Snow Labs name: legclf_cuad_indemnifications_clause date: 2022-09-27 tags: [cuad, indemnifications, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `indemnifications` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `indemnifications` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_indemnifications_clause_en_1.0.0_3.0_1664272531526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_indemnifications_clause_en_1.0.0_3.0_1664272531526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cuad_indemnifications_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[indemnifications]| |[other]| |[other]| |[indemnifications]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_indemnifications_clause| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|21.9 MB| ## References In-house annotations on CUAD dataset ## Benchmarking ```bash label precision recall f1-score support indemnifications 1.00 0.83 0.91 12 other 0.83 1.00 0.91 10 accuracy - - 0.91 22 macro avg 0.92 0.92 0.91 22 weighted avg 0.92 0.91 0.91 22 ``` --- layout: model title: English BertForMaskedLM Large Cased model author: John Snow Labs name: bert_embeddings_large_cased_whole_word_masking date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-cased-whole-word-masking` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_cased_whole_word_masking_en_4.2.4_3.0_1670020123161.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_cased_whole_word_masking_en_4.2.4_3.0_1670020123161.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_cased_whole_word_masking","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_cased_whole_word_masking","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_cased_whole_word_masking| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/bert-large-cased-whole-word-masking - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English RobertaForQuestionAnswering (from comacrae) author: John Snow Labs name: roberta_qa_roberta_eda_and_parav3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-eda-and-parav3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_eda_and_parav3_en_4.0.0_3.0_1655735762543.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_eda_and_parav3_en_4.0.0_3.0_1655735762543.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_eda_and_parav3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_eda_and_parav3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.eda_and_parav3.by_comacrae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_eda_and_parav3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/comacrae/roberta-eda-and-parav3 --- layout: model title: Part of Speech for Bulgarian author: John Snow Labs name: pos_ud_btb date: 2021-03-08 tags: [part_of_speech, open_source, bulgarian, pos_ud_btb, bg] task: Part of Speech Tagging language: bg edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADP - NOUN - PUNCT - VERB - AUX - PRON - ADJ - PART - ADV - INTJ - DET - PROPN - CCONJ - NUM - SCONJ - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_btb_bg_3.0.0_3.0_1615230275121.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_btb_bg_3.0.0_3.0_1615230275121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_btb", "bg") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Здравейте от Lak Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_btb", "bg") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Здравейте от Lak Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Здравейте от Lak Snow Labs! ""] token_df = nlu.load('bg.pos.ud_btb').predict(text) token_df ```

## Results ```bash token pos 0 Здравейте VERB 1 от ADP 2 Lak ADJ 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_btb| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|bg| --- layout: model title: Pipeline for Detect Medication author: John Snow Labs name: ner_medication_pipeline date: 2022-07-28 tags: [ner, en, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained pipeline to detect medication entities. It was built on the top of `ner_posology_greedy` model and also augmented with the drug names mentioned in UK and US drugbank datasets. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_medication_pipeline_en_4.0.0_3.0_1658987434372.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_medication_pipeline_en_4.0.0_3.0_1658987434372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_medication_pipeline = PretrainedPipeline("ner_medication_pipeline", "en", "clinical/models") text = """The patient was prescribed metformin 1000 MG, and glipizide 2.5 MG. The other patient was given Fragmin 5000 units, Xenaderm to wounds topically b.i.d. and OxyContin 30 mg.""" result = ner_medication_pipeline.fullAnnotate([text]) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_medication_pipeline = new PretrainedPipeline("ner_medication_pipeline", "en", "clinical/models") val result = ner_medication_pipeline.fullAnnotate("The patient was prescribed metformin 1000 MG, and glipizide 2.5 MG. The other patient was given Fragmin 5000 units, Xenaderm to wounds topically b.i.d. and OxyContin 30 mg."") ``` {:.nlu-block} ```python | ner_chunk | entity | |:-------------------|:---------| | metformin 1000 MG | DRUG | | glipizide 2.5 MG | DRUG | | Fragmin 5000 units | DRUG | | Xenaderm | DRUG | | OxyContin 30 mg | DRUG | ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_medication_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - TextMatcherModel - ChunkMergeModel - Finisher --- layout: model title: ALBERT Embeddings (Large Uncase) author: John Snow Labs name: albert_large_uncased date: 2020-04-28 task: Embeddings language: en nav_key: models edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ALBERT is "A Lite" version of BERT, a popular unsupervised language representation learning algorithm. ALBERT uses parameter-reduction techniques that allow for large-scale configurations, overcome previous memory limitations, and achieve better behavior with respect to model degradation. The details are described in the paper "[ALBERT: A Lite BERT for Self-supervised Learning of Language Representations.](https://arxiv.org/abs/1909.11942)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_large_uncased_en_2.5.0_2.4_1588073397355.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_large_uncased_en_2.5.0_2.4_1588073397355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = AlbertEmbeddings.pretrained("albert_large_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = AlbertEmbeddings.pretrained("albert_large_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.albert.large_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_albert_large_uncased_embeddings I [0.3967159688472748, -0.6448764801025391, -0.3... love [1.1107065677642822, -0.2454298734664917, 0.60... NLP [0.02937467396259308, -0.7092287540435791, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_large_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/albert_large/3](https://tfhub.dev/google/albert_large/3) --- layout: model title: Stop Words Cleaner for Russian author: John Snow Labs name: stopwords_ru date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: ru edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, ru] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_ru_ru_2.5.4_2.4_1594742439248.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_ru_ru_2.5.4_2.4_1594742439248.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_ru", "ru") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_ru", "ru") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены."""] stopword_df = nlu.load('ru.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=5, result='Помимо', metadata={'sentence': '0'}), Row(annotatorType='token', begin=11, end=11, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=25, result='король', metadata={'sentence': '0'}), Row(annotatorType='token', begin=27, end=32, result='севера', metadata={'sentence': '0'}), Row(annotatorType='token', begin=33, end=33, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_ru| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ru| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Arabic Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_bert_large_arabic date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-arabic` is a Arabic model orginally trained by `asafaya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_arabic_ar_3.4.2_3.0_1649678414101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_arabic_ar_3.4.2_3.0_1649678414101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_arabic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/asafaya/bert-large-arabic - https://traces1.inria.fr/oscar/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/bert - https://www.tensorflow.org/tfrc - https://github.com/alisafaya/Arabic-BERT --- layout: model title: Hindi BertForQuestionAnswering model (from Sindhu) author: John Snow Labs name: bert_qa_muril_large_squad2 date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: hi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `muril-large-squad2` is a Hindi model orginally trained by `Sindhu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_muril_large_squad2_hi_4.0.0_3.0_1654188807056.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_muril_large_squad2_hi_4.0.0_3.0_1654188807056.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_muril_large_squad2","hi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_muril_large_squad2","hi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("hi.answer_question.squadv2.bert.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_muril_large_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hi| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Sindhu/muril-large-squad2 - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://twitter.com/batw0man --- layout: model title: Google's Tapas Table Understanding (Medium, WIKISQL) author: John Snow Labs name: table_qa_tapas_medium_finetuned_wikisql_supervised date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Medium Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_medium_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530746170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_medium_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530746170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_medium_finetuned_wikisql_supervised","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_autotrain_dog_vs_food", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_autotrain_dog_vs_food", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_autotrain_dog_vs_food| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: roberta_qa_deepset_base_squad2_orkg_no_label_5e_05 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deepset-roberta-base-squad2-orkg-no-label-5e-05` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_no_label_5e_05_en_4.3.0_3.0_1674209664953.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_no_label_5e_05_en_4.3.0_3.0_1674209664953.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_no_label_5e_05","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_no_label_5e_05","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_base_squad2_orkg_no_label_5e_05| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Moussab/deepset-roberta-base-squad2-orkg-no-label-5e-05 --- layout: model title: English BertForQuestionAnswering model (from HankyStyle) author: John Snow Labs name: bert_qa_Multi_ling_BERT date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Multi-ling-BERT` is a English model orginally trained by `HankyStyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Multi_ling_BERT_en_4.0.0_3.0_1654178859506.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Multi_ling_BERT_en_4.0.0_3.0_1654178859506.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Multi_ling_BERT","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Multi_ling_BERT","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_HankyStyle").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Multi_ling_BERT| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|626.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/HankyStyle/Multi-ling-BERT --- layout: model title: Word2Vec Embeddings in Swahili (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, sw, open_source] task: Embeddings language: sw edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sw_3.4.1_3.0_1647459595377.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sw_3.4.1_3.0_1647459595377.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mysquadv2_8Jan22_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mysquadv2_8Jan22_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.distil_bert.v2.by_threem").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_mysquadv2_8Jan22_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/threem/mysquadv2_8Jan22-finetuned-squad --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from ArneD) author: John Snow Labs name: xlmroberta_ner_arned_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `ArneD`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_arned_base_finetuned_panx_de_4.1.0_3.0_1660429307551.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_arned_base_finetuned_panx_de_4.1.0_3.0_1660429307551.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_arned_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_arned_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_arned_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ArneD/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_anan0329 TFWav2Vec2ForCTC from anan0329 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_anan0329 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_anan0329` is a English model originally trained by anan0329. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_anan0329_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_anan0329_en_4.2.0_3.0_1664114661625.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_anan0329_en_4.2.0_3.0_1664114661625.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_anan0329", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_anan0329", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_anan0329| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: NER Pipeline for Tests - Voice of the Patient author: John Snow Labs name: ner_vop_test_pipeline date: 2023-06-10 tags: [licensed, pipeline, ner, en, vop, test] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline extracts mentions of tests and their results from health-related text in colloquial language. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_pipeline_en_4.4.3_3.0_1686427000395.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_pipeline_en_4.4.3_3.0_1686427000395.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_vop_test_pipeline", "en", "clinical/models") pipeline.annotate(" I went to the endocrinology department to get my thyroid levels checked. They ordered a blood test and found out that I have hypothyroidism, so now I'm on medication to manage it. ") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_vop_test_pipeline", "en", "clinical/models") val result = pipeline.annotate(" I went to the endocrinology department to get my thyroid levels checked. They ordered a blood test and found out that I have hypothyroidism, so now I'm on medication to manage it. ") ```

## Results ```bash | chunk | ner_label | |:---------------|:------------| | thyroid levels | Test | | blood test | Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_test_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|791.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Legal Representations Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_representations_bert date: 2023-03-05 tags: [en, legal, classification, clauses, representations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Representations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Representations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_representations_bert_en_1.0.0_3.0_1678050021027.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_representations_bert_en_1.0.0_3.0_1678050021027.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_representations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Representations]| |[Other]| |[Other]| |[Representations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_representations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.92 0.91 0.92 93 Representations 0.88 0.89 0.89 66 accuracy - - 0.91 159 macro-avg 0.90 0.90 0.90 159 weighted-avg 0.91 0.91 0.91 159 ``` --- layout: model title: RxNorm to MeSH Code Mapping author: John Snow Labs name: rxnorm_mesh_mapping date: 2023-06-13 tags: [rxnorm, mesh, en, licensed, pipeline] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps RxNorm codes to MeSH codes without using any text data. You’ll just feed white space-delimited RxNorm codes and it will return the corresponding MeSH codes as a list. If there is no mapping, the original code is returned with no mapping. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_mesh_mapping_en_4.4.4_3.2_1686663529810.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_mesh_mapping_en_4.4.4_3.2_1686663529810.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rxnorm_mesh_mapping","en","clinical/models") pipeline.annotate("1191 6809 47613") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rxnorm_mesh_mapping","en","clinical/models") val result = pipeline.annotate("1191 6809 47613") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.mesh").predict("""1191 6809 47613""") ```

## Results ```bash Results {'rxnorm': ['1191', '6809', '47613'], 'mesh': ['D001241', 'D008687', 'D019355']} Note: | RxNorm | Details | | ---------- | -------------------:| | 1191 | aspirin | | 6809 | metformin | | 47613 | calcium citrate | | MeSH | Details | | ---------- | -------------------:| | D001241 | Aspirin | | D008687 | Metformin | | D019355 | Calcium Citrate | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_mesh_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|103.6 KB| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: T5 for Active to Passive Style Transfer author: John Snow Labs name: t5_active_to_passive_styletransfer date: 2022-01-12 tags: [t5, open_source, en] task: Text Generation language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a text-to-text model based on T5 fine-tuned to generate actively written text from a passively written text input, for the task "transfer Active to Passive:". It is based on Prithiviraj Damodaran's Styleformer. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5_LINGUISTIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_active_to_passive_styletransfer_en_3.4.0_3.0_1641987400533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_active_to_passive_styletransfer_en_3.4.0_3.0_1641987400533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from sparknlp.base import * from sparknlp.annotator import * spark = sparknlp.start() documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = T5Transformer.pretrained("t5_active_to_passive_styletransfer") \ .setTask("transfer Active to Passive:") \ .setInputCols(["documents"]) \ .setMaxOutputLength(200) \ .setOutputCol("transfers") pipeline = Pipeline().setStages([documentAssembler, t5]) data = spark.createDataFrame([["I am writing you a letter."]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("transfers.result").show(truncate=False) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.seq2seq.T5Transformer import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer.pretrained("t5_active_to_passive_styletransfer") .setTask("transfer Active to Passive:") .setMaxOutputLength(200) .setInputCols("documents") .setOutputCol("transfer") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("I am writing you a letter.").toDF("text") val result = pipeline.fit(data).transform(data) result.select("transfer.result").show(false) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5.active_to_passive_styletransfer").predict("""transfer Active to Passive:""") ```

## Results ```bash +---------------------------+ |result | +---------------------------+ |[a letter is written by me]| +---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_active_to_passive_styletransfer| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[transfers]| |Language:|en| |Size:|264.5 MB| ## Data Source The original model is from the transformers library: https://huggingface.co/prithivida/active_to_passive_styletransfer --- layout: model title: Luo (Kenya and Tanzania) XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_ner date: 2022-08-01 tags: [luo, open_source, xlm_roberta, ner] task: Named Entity Recognition language: luo edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-ner-luo` is a Luo (Kenya and Tanzania) model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_ner_luo_4.1.0_3.0_1659355137886.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_ner_luo_4.1.0_3.0_1659355137886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_ner","luo") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_ner","luo") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_ner| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|luo| |Size:|772.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-ner-luo - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Pipeline to Detect Anatomical and Observation Entities in Chest Radiology Reports (CheXpert) author: John Snow Labs name: ner_chexpert_pipeline date: 2023-03-14 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chexpert](https://nlp.johnsnowlabs.com/2021/09/30/ner_chexpert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_pipeline_en_4.3.0_3.2_1678779791404.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_pipeline_en_4.3.0_3.2_1678779791404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_chexpert_pipeline", "en", "clinical/models") text = '''FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_chexpert_pipeline", "en", "clinical/models") val text = "FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chexpert.pipeline").predict("""FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax. FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:------------|-------------:| | 0 | endotracheal | 118 | 129 | OBS | 0.9881 | | 1 | tube | 131 | 134 | OBS | 0.9996 | | 2 | Swan - Ganz | 140 | 150 | OBS | 0.9625 | | 3 | catheter | 152 | 159 | OBS | 0.9919 | | 4 | left | 185 | 188 | ANAT | 0.9983 | | 5 | chest | 190 | 194 | ANAT | 0.9749 | | 6 | tube | 196 | 199 | OBS | 0.9999 | | 7 | in place | 209 | 216 | OBS | 0.9894 | | 8 | pneumothorax | 246 | 257 | OBS | 0.9997 | | 9 | Mild | 260 | 263 | OBS | 0.9988 | | 10 | atelectatic | 265 | 275 | OBS | 0.9986 | | 11 | changes | 277 | 283 | OBS | 0.9984 | | 12 | left | 301 | 304 | ANAT | 0.9999 | | 13 | base | 306 | 309 | ANAT | 0.9999 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chexpert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Arabic ALBERT Embeddings (Large) author: John Snow Labs name: albert_embeddings_albert_large_arabic date: 2022-04-14 tags: [albert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-large-arabic` is a Arabic model orginally trained by `asafaya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_large_arabic_ar_3.4.2_3.0_1649954278357.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_large_arabic_ar_3.4.2_3.0_1649954278357.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_large_arabic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|68.0 MB| |Case sensitive:|false| ## References - https://huggingface.co/asafaya/albert-large-arabic - https://oscar-corpus.com/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/albert - https://www.tensorflow.org/tfrc - https://github.com/KUIS-AI-Lab/Arabic-ALBERT/ --- layout: model title: Legal Release Clause Binary Classifier author: John Snow Labs name: legclf_release_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `release` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `release` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_release_clause_en_1.0.0_3.2_1660122916319.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_release_clause_en_1.0.0_3.2_1660122916319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_release_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[release]| |[other]| |[other]| |[release]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_release_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.88 1.00 0.93 70 release 1.00 0.50 0.67 20 accuracy - - 0.89 90 macro-avg 0.94 0.75 0.80 90 weighted-avg 0.90 0.89 0.87 90 ``` --- layout: model title: Toxic Comment Classification - Small author: John Snow Labs name: multiclassifierdl_use_toxic_sm date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, en, text_classification] supported: true annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Discussing things you care about can be difficult. The threat of abuse and harassment online means that many people stop expressing themselves and give up on seeking different opinions. Platforms struggle to effectively facilitate conversations, leading many communities to limit or completely shut down user comments. The Conversation AI team, a research initiative founded by Jigsaw and Google (both a part of Alphabet) is working on tools to help improve the online conversation. One area of focus is the study of negative online behaviors, like toxic comments (i.e. comments that are rude, disrespectful, or otherwise likely to make someone leave a discussion). So far they’ve built a range of publicly available models served through the Perspective API, including toxicity. But the current models still make errors, and they don’t allow users to select which types of toxicity they’re interested in finding (e.g. some platforms may be fine with profanity, but not with other types of toxic content). Automatically detect identity hate, insult, obscene, severe toxic, threat, or toxic content in SM comments using our out-of-the-box Spark NLP Multiclassifier DL. We removed the records without any labels in this model. (only 14K+ comments were used to train this model) ## Predicted Entities `toxic`, `severe_toxic`, `identity_hate`, `insult`, `obscene`, `threat` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_MULTILABEL_TOXIC/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_MULTILABEL_TOXIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/multiclassifierdl_use_toxic_sm_en_2.7.1_2.4_1611230645484.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/multiclassifierdl_use_toxic_sm_en_2.7.1_2.4_1611230645484.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained() \ .setInputCols(["document"])\ .setOutputCol("use_embeddings") docClassifier = MultiClassifierDLModel.pretrained("multiclassifierdl_use_toxic_sm") \ .setInputCols(["use_embeddings"])\ .setOutputCol("category")\ .setThreshold(0.5) pipeline = Pipeline( stages = [ document, use, docClassifier ]) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") .setCleanupMode("shrink") val use = UniversalSentenceEncoder.pretrained() .setInputCols("document") .setOutputCol("use_embeddings") val docClassifier = MultiClassifierDLModel.pretrained("multiclassifierdl_use_toxic_sm") .setInputCols("use_embeddings") .setOutputCol("category") .setThreshold(0.5f) val pipeline = new Pipeline() .setStages( Array( documentAssembler, use, docClassifier ) ) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.toxic.sm").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|multiclassifierdl_use_toxic_sm| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[use_embeddings]| |Output Labels:|[category]| |Language:|en| ## Data Source https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/overview ## Benchmarking ```bash Classification report: precision recall f1-score support 0 0.56 0.30 0.39 127 1 0.71 0.70 0.70 761 2 0.76 0.72 0.74 824 3 0.55 0.21 0.31 147 4 0.79 0.38 0.51 50 5 0.94 1.00 0.97 1504 micro avg 0.83 0.80 0.81 3413 macro avg 0.72 0.55 0.60 3413 weighted avg 0.81 0.80 0.80 3413 samples avg 0.84 0.83 0.80 3413 ``` --- layout: model title: Detect PHI for Generic Deidentification in Romanian (BERT) author: John Snow Labs name: ner_deid_generic_bert date: 2022-11-22 tags: [licensed, clinical, ro, deidentification, phi, generic, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators to allow a generic model to be trained by using a Deep Learning architecture (Char CNN's - BiLSTM - CRF - word embeddings) inspired by a former state-of-the-art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It is trained with bert_base_cased embeddings and can detect 7 generic entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_4.2.2_3.0_1669122326582.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_4.2.2_3.0_1669122326582.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|LOCATION | |Drumul Oprea Nr |LOCATION | |972 |LOCATION | |Vaslui |LOCATION | |737405 |LOCATION | |+40(235)413773 |CONTACT | |25 May 2022 |DATE | |BUREAN MARIA |NAME | |77 |AGE | |Agota Evelyn Tımar |NAME | |2450502264401 |ID | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_bert| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.5 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.95 0.97 0.96 1186 CONTACT 0.99 0.98 0.98 366 DATE 0.96 0.92 0.94 4518 ID 1.00 1.00 1.00 679 LOCATION 0.91 0.90 0.90 1683 NAME 0.93 0.96 0.94 2916 PROFESSION 0.87 0.85 0.86 161 micro-avg 0.94 0.94 0.94 11509 macro-avg 0.94 0.94 0.94 11509 weighted-avg 0.95 0.94 0.94 11509 ``` --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_greek_2 TFWav2Vec2ForCTC from skylord author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_greek_2 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_greek_2` is a Modern Greek (1453-) model originally trained by skylord. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_greek_2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_greek_2_el_4.2.0_3.0_1664112085138.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_greek_2_el_4.2.0_3.0_1664112085138.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_greek_2', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_greek_2", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_greek_2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_base_finetuned_deletion_squad_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-deletion-squad-10` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_deletion_squad_10_en_4.3.0_3.0_1674216541297.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_deletion_squad_10_en_4.3.0_3.0_1674216541297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_deletion_squad_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_deletion_squad_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_deletion_squad_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-deletion-squad-10 --- layout: model title: English image_classifier_vit_demo ViTForImageClassification from nguyenbh author: John Snow Labs name: image_classifier_vit_demo date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_demo` is a English model originally trained by nguyenbh. ## Predicted Entities `turnstile`, `damselfly`, `mixing bowl`, `sea snake`, `cockroach, roach`, `buckle`, `beer glass`, `bulbul`, `lumbermill, sawmill`, `whippet`, `Australian terrier`, `television, television system`, `hoopskirt, crinoline`, `horse cart, horse-cart`, `guillotine`, `malamute, malemute, Alaskan malamute`, `coyote, prairie wolf, brush wolf, Canis latrans`, `colobus, colobus monkey`, `hognose snake, puff adder, sand viper`, `sock`, `burrito`, `printer`, `bathing cap, swimming cap`, `chiton, coat-of-mail shell, sea cradle, polyplacophore`, `Rottweiler`, `cello, violoncello`, `pitcher, ewer`, `computer keyboard, keypad`, `bow`, `peacock`, `ballplayer, baseball player`, `refrigerator, icebox`, `solar dish, solar collector, solar furnace`, `passenger car, coach, carriage`, `African chameleon, Chamaeleo chamaeleon`, `oboe, hautboy, hautbois`, `toyshop`, `Leonberg`, `howler monkey, howler`, `bluetick`, `African elephant, Loxodonta africana`, `American lobster, Northern lobster, Maine lobster, Homarus americanus`, `combination lock`, `black-and-tan coonhound`, `bonnet, poke bonnet`, `harvester, reaper`, `Appenzeller`, `iron, smoothing iron`, `electric locomotive`, `lycaenid, lycaenid butterfly`, `sandbar, sand bar`, `Cardigan, Cardigan Welsh corgi`, `pencil sharpener`, `jean, blue jean, denim`, `backpack, back pack, knapsack, packsack, rucksack, haversack`, `monitor`, `ice cream, icecream`, `apiary, bee house`, `water jug`, `American coot, marsh hen, mud hen, water hen, Fulica americana`, `ground beetle, carabid beetle`, `jigsaw puzzle`, `ant, emmet, pismire`, `wreck`, `kuvasz`, `gyromitra`, `Ibizan hound, Ibizan Podenco`, `brown bear, bruin, Ursus arctos`, `bolo tie, bolo, bola tie, bola`, `Pembroke, Pembroke Welsh corgi`, `French bulldog`, `prison, prison house`, `ballpoint, ballpoint pen, ballpen, Biro`, `stage`, `airliner`, `dogsled, dog sled, dog sleigh`, `redshank, Tringa totanus`, `menu`, `Indian cobra, Naja naja`, `swab, swob, mop`, `window screen`, `brain coral`, `artichoke, globe artichoke`, `loupe, jeweler's loupe`, `loudspeaker, speaker, speaker unit, loudspeaker system, speaker system`, `panpipe, pandean pipe, syrinx`, `wok`, `croquet ball`, `plate`, `scoreboard`, `Samoyed, Samoyede`, `ocarina, sweet potato`, `beaver`, `borzoi, Russian wolfhound`, `horizontal bar, high bar`, `stretcher`, `seat belt, seatbelt`, `obelisk`, `forklift`, `feather boa, boa`, `frying pan, frypan, skillet`, `barbershop`, `hamper`, `face powder`, `Siamese cat, Siamese`, `ladle`, `dingo, warrigal, warragal, Canis dingo`, `mountain tent`, `head cabbage`, `echidna, spiny anteater, anteater`, `Polaroid camera, Polaroid Land camera`, `dumbbell`, `espresso`, `notebook, notebook computer`, `Norfolk terrier`, `binoculars, field glasses, opera glasses`, `carpenter's kit, tool kit`, `moving van`, `catamaran`, `tiger beetle`, `bikini, two-piece`, `Siberian husky`, `studio couch, day bed`, `bulletproof vest`, `lawn mower, mower`, `promontory, headland, head, foreland`, `soap dispenser`, `vulture`, `dam, dike, dyke`, `brambling, Fringilla montifringilla`, `toilet tissue, toilet paper, bathroom tissue`, `ringlet, ringlet butterfly`, `tiger cat`, `mobile home, manufactured home`, `Norwich terrier`, `little blue heron, Egretta caerulea`, `English setter`, `Tibetan mastiff`, `rocking chair, rocker`, `mask`, `maze, labyrinth`, `bookcase`, `viaduct`, `sweatshirt`, `plow, plough`, `basenji`, `typewriter keyboard`, `Windsor tie`, `coral fungus`, `desktop computer`, `Kerry blue terrier`, `Angora, Angora rabbit`, `can opener, tin opener`, `shield, buckler`, `triumphal arch`, `horned viper, cerastes, sand viper, horned asp, Cerastes cornutus`, `miniature schnauzer`, `tape player`, `jaguar, panther, Panthera onca, Felis onca`, `hook, claw`, `file, file cabinet, filing cabinet`, `chime, bell, gong`, `shower curtain`, `window shade`, `acoustic guitar`, `gas pump, gasoline pump, petrol pump, island dispenser`, `cicada, cicala`, `Petri dish`, `paintbrush`, `banana`, `chickadee`, `mountain bike, all-terrain bike, off-roader`, `lighter, light, igniter, ignitor`, `oil filter`, `cab, hack, taxi, taxicab`, `Christmas stocking`, `rugby ball`, `black widow, Latrodectus mactans`, `bustard`, `fiddler crab`, `web site, website, internet site, site`, `chocolate sauce, chocolate syrup`, `chainlink fence`, `fireboat`, `cocktail shaker`, `airship, dirigible`, `projectile, missile`, `bagel, beigel`, `screwdriver`, `oystercatcher, oyster catcher`, `pot, flowerpot`, `water bottle`, `Loafer`, `drumstick`, `soccer ball`, `cairn, cairn terrier`, `padlock`, `tow truck, tow car, wrecker`, `bloodhound, sleuthhound`, `punching bag, punch bag, punching ball, punchball`, `great grey owl, great gray owl, Strix nebulosa`, `scale, weighing machine`, `trench coat`, `briard`, `cheetah, chetah, Acinonyx jubatus`, `entertainment center`, `Boston bull, Boston terrier`, `Arabian camel, dromedary, Camelus dromedarius`, `steam locomotive`, `coil, spiral, volute, whorl, helix`, `plane, carpenter's plane, woodworking plane`, `gondola`, `spider web, spider's web`, `bathtub, bathing tub, bath, tub`, `pelican`, `miniature poodle`, `cowboy boot`, `perfume, essence`, `lakeside, lakeshore`, `timber wolf, grey wolf, gray wolf, Canis lupus`, `moped`, `sunscreen, sunblock, sun blocker`, `Brabancon griffon`, `puffer, pufferfish, blowfish, globefish`, `lifeboat`, `pool table, billiard table, snooker table`, `Bouvier des Flandres, Bouviers des Flandres`, `Pomeranian`, `theater curtain, theatre curtain`, `marimba, xylophone`, `baboon`, `vacuum, vacuum cleaner`, `pill bottle`, `pick, plectrum, plectron`, `hen`, `American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier`, `digital watch`, `pier`, `oxygen mask`, `Tibetan terrier, chrysanthemum dog`, `ostrich, Struthio camelus`, `water ouzel, dipper`, `drilling platform, offshore rig`, `magnetic compass`, `throne`, `butternut squash`, `minibus`, `EntleBucher`, `carousel, carrousel, merry-go-round, roundabout, whirligig`, `hot pot, hotpot`, `rain barrel`, `wood rabbit, cottontail, cottontail rabbit`, `miniature pinscher`, `partridge`, `three-toed sloth, ai, Bradypus tridactylus`, `English springer, English springer spaniel`, `corkscrew, bottle screw`, `fur coat`, `robin, American robin, Turdus migratorius`, `dowitcher`, `ruddy turnstone, Arenaria interpres`, `water snake`, `stove`, `Great Pyrenees`, `soft-coated wheaten terrier`, `carbonara`, `snail`, `breastplate, aegis, egis`, `wolf spider, hunting spider`, `hatchet`, `CD player`, `axolotl, mud puppy, Ambystoma mexicanum`, `pomegranate`, `poncho`, `leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea`, `lorikeet`, `spatula`, `jay`, `platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus`, `stethoscope`, `flagpole, flagstaff`, `coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch`, `agama`, `red wolf, maned wolf, Canis rufus, Canis niger`, `beaker`, `eft`, `pretzel`, `brassiere, bra, bandeau`, `frilled lizard, Chlamydosaurus kingi`, `joystick`, `goldfish, Carassius auratus`, `fig`, `maypole`, `caldron, cauldron`, `admiral`, `impala, Aepyceros melampus`, `spotted salamander, Ambystoma maculatum`, `syringe`, `hog, pig, grunter, squealer, Sus scrofa`, `handkerchief, hankie, hanky, hankey`, `tarantula`, `cheeseburger`, `pinwheel`, `sax, saxophone`, `dung beetle`, `broccoli`, `cassette player`, `milk can`, `traffic light, traffic signal, stoplight`, `shovel`, `sarong`, `tabby, tabby cat`, `parallel bars, bars`, `ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle`, `quill, quill pen`, `giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca`, `steel drum`, `quail`, `Blenheim spaniel`, `wig`, `hamster`, `ice lolly, lolly, lollipop, popsicle`, `seashore, coast, seacoast, sea-coast`, `chest`, `worm fence, snake fence, snake-rail fence, Virginia fence`, `missile`, `beer bottle`, `yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum`, `breakwater, groin, groyne, mole, bulwark, seawall, jetty`, `white wolf, Arctic wolf, Canis lupus tundrarum`, `guacamole`, `porcupine, hedgehog`, `trolleybus, trolley coach, trackless trolley`, `greenhouse, nursery, glasshouse`, `trimaran`, `Italian greyhound`, `potter's wheel`, `jacamar`, `wallet, billfold, notecase, pocketbook`, `Lakeland terrier`, `green lizard, Lacerta viridis`, `indigo bunting, indigo finch, indigo bird, Passerina cyanea`, `green mamba`, `walking stick, walkingstick, stick insect`, `crossword puzzle, crossword`, `eggnog`, `barrow, garden cart, lawn cart, wheelbarrow`, `remote control, remote`, `bicycle-built-for-two, tandem bicycle, tandem`, `wool, woolen, woollen`, `black grouse`, `abaya`, `marmoset`, `golf ball`, `jeep, landrover`, `Mexican hairless`, `dishwasher, dish washer, dishwashing machine`, `jersey, T-shirt, tee shirt`, `planetarium`, `goose`, `mailbox, letter box`, `capuchin, ringtail, Cebus capucinus`, `marmot`, `orangutan, orang, orangutang, Pongo pygmaeus`, `coffeepot`, `ambulance`, `shopping basket`, `pop bottle, soda bottle`, `red fox, Vulpes vulpes`, `crash helmet`, `street sign`, `affenpinscher, monkey pinscher, monkey dog`, `Arctic fox, white fox, Alopex lagopus`, `sidewinder, horned rattlesnake, Crotalus cerastes`, `ruffed grouse, partridge, Bonasa umbellus`, `muzzle`, `measuring cup`, `canoe`, `reflex camera`, `fox squirrel, eastern fox squirrel, Sciurus niger`, `French loaf`, `killer whale, killer, orca, grampus, sea wolf, Orcinus orca`, `dial telephone, dial phone`, `thimble`, `bubble`, `vizsla, Hungarian pointer`, `running shoe`, `mailbag, postbag`, `radio telescope, radio reflector`, `piggy bank, penny bank`, `Chihuahua`, `chambered nautilus, pearly nautilus, nautilus`, `Airedale, Airedale terrier`, `kimono`, `green snake, grass snake`, `rubber eraser, rubber, pencil eraser`, `upright, upright piano`, `orange`, `revolver, six-gun, six-shooter`, `ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin`, `drum, membranophone, tympan`, `Dungeness crab, Cancer magister`, `lipstick, lip rouge`, `gong, tam-tam`, `fountain`, `tub, vat`, `malinois`, `sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita`, `German short-haired pointer`, `apron`, `Irish setter, red setter`, `dishrag, dishcloth`, `school bus`, `candle, taper, wax light`, `bib`, `cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM`, `power drill`, `English foxhound`, `miniskirt, mini`, `swing`, `slug`, `hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa`, `rifle`, `Saluki, gazelle hound`, `Sealyham terrier, Sealyham`, `bullet train, bullet`, `hyena, hyaena`, `ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus`, `toy terrier`, `goblet`, `safe`, `cup`, `electric guitar`, `red wine`, `restaurant, eating house, eating place, eatery`, `wall clock`, `washbasin, handbasin, washbowl, lavabo, wash-hand basin`, `red-breasted merganser, Mergus serrator`, `crate`, `banded gecko`, `hippopotamus, hippo, river horse, Hippopotamus amphibius`, `tick`, `tripod`, `sombrero`, `desk`, `sea slug, nudibranch`, `racer, race car, racing car`, `pizza, pizza pie`, `dining table, board`, `Saint Bernard, St Bernard`, `komondor`, `electric ray, crampfish, numbfish, torpedo`, `prairie chicken, prairie grouse, prairie fowl`, `coffee mug`, `hammer`, `golfcart, golf cart`, `unicycle, monocycle`, `bison`, `soup bowl`, `rapeseed`, `golden retriever`, `plastic bag`, `grey fox, gray fox, Urocyon cinereoargenteus`, `water tower`, `house finch, linnet, Carpodacus mexicanus`, `barbell`, `hair slide`, `tiger, Panthera tigris`, `black-footed ferret, ferret, Mustela nigripes`, `meat loaf, meatloaf`, `hand blower, blow dryer, blow drier, hair dryer, hair drier`, `overskirt`, `gibbon, Hylobates lar`, `Gila monster, Heloderma suspectum`, `toucan`, `snowmobile`, `pencil box, pencil case`, `scuba diver`, `cloak`, `Sussex spaniel`, `otter`, `Greater Swiss Mountain dog`, `great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias`, `torch`, `magpie`, `tiger shark, Galeocerdo cuvieri`, `wing`, `Border collie`, `bell cote, bell cot`, `sea anemone, anemone`, `teapot`, `sea urchin`, `screen, CRT screen`, `bookshop, bookstore, bookstall`, `oscilloscope, scope, cathode-ray oscilloscope, CRO`, `crib, cot`, `police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria`, `hartebeest`, `manhole cover`, `iPod`, `rock python, rock snake, Python sebae`, `nipple`, `suspension bridge`, `safety pin`, `sea lion`, `cougar, puma, catamount, mountain lion, painter, panther, Felis concolor`, `mantis, mantid`, `wardrobe, closet, press`, `projector`, `Granny Smith`, `diamondback, diamondback rattlesnake, Crotalus adamanteus`, `pirate, pirate ship`, `espresso maker`, `African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus`, `cradle`, `common newt, Triturus vulgaris`, `tricycle, trike, velocipede`, `bobsled, bobsleigh, bob`, `thunder snake, worm snake, Carphophis amoenus`, `thresher, thrasher, threshing machine`, `banjo`, `armadillo`, `pajama, pyjama, pj's, jammies`, `ski`, `Maltese dog, Maltese terrier, Maltese`, `leafhopper`, `book jacket, dust cover, dust jacket, dust wrapper`, `silky terrier, Sydney silky`, `Shih-Tzu`, `wallaby, brush kangaroo`, `cardigan`, `sturgeon`, `freight car`, `home theater, home theatre`, `sundial`, `African crocodile, Nile crocodile, Crocodylus niloticus`, `odometer, hodometer, mileometer, milometer`, `sliding door`, `vine snake`, `West Highland white terrier`, `mongoose`, `hornbill`, `beagle`, `European gallinule, Porphyrio porphyrio`, `submarine, pigboat, sub, U-boat`, `Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis`, `cock`, `pedestal, plinth, footstall`, `accordion, piano accordion, squeeze box`, `gown`, `lynx, catamount`, `guenon, guenon monkey`, `Walker hound, Walker foxhound`, `standard schnauzer`, `reel`, `hip, rose hip, rosehip`, `grasshopper, hopper`, `Dutch oven`, `stone wall`, `hard disc, hard disk, fixed disk`, `snow leopard, ounce, Panthera uncia`, `shopping cart`, `digital clock`, `hourglass`, `Border terrier`, `Old English sheepdog, bobtail`, `academic gown, academic robe, judge's robe`, `spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish`, `spotlight, spot`, `dome`, `barn spider, Araneus cavaticus`, `bee eater`, `basketball`, `cliff dwelling`, `folding chair`, `isopod`, `Doberman, Doberman pinscher`, `bittern`, `sunglasses, dark glasses, shades`, `picket fence, paling`, `Crock Pot`, `ibex, Capra ibex`, `neck brace`, `cardoon`, `cassette`, `amphibian, amphibious vehicle`, `minivan`, `analog clock`, `trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi`, `yurt`, `cliff, drop, drop-off`, `Bernese mountain dog`, `teddy, teddy bear`, `sloth bear, Melursus ursinus, Ursus ursinus`, `bassoon`, `toaster`, `ptarmigan`, `Gordon setter`, `night snake, Hypsiglena torquata`, `grand piano, grand`, `purse`, `clumber, clumber spaniel`, `shoji`, `hair spray`, `maillot`, `knee pad`, `space heater`, `bottlecap`, `chiffonier, commode`, `chain saw, chainsaw`, `sulphur butterfly, sulfur butterfly`, `pay-phone, pay-station`, `kelpie`, `mouse, computer mouse`, `car wheel`, `cornet, horn, trumpet, trump`, `container ship, containership, container vessel`, `matchstick`, `scabbard`, `American black bear, black bear, Ursus americanus, Euarctos americanus`, `langur`, `rock crab, Cancer irroratus`, `lionfish`, `speedboat`, `black stork, Ciconia nigra`, `knot`, `disk brake, disc brake`, `mosquito net`, `white stork, Ciconia ciconia`, `abacus`, `titi, titi monkey`, `grocery store, grocery, food market, market`, `waffle iron`, `pickelhaube`, `wooden spoon`, `Norwegian elkhound, elkhound`, `earthstar`, `sewing machine`, `balance beam, beam`, `potpie`, `chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour`, `Staffordshire bullterrier, Staffordshire bull terrier`, `switch, electric switch, electrical switch`, `dhole, Cuon alpinus`, `paddle, boat paddle`, `limousine, limo`, `Shetland sheepdog, Shetland sheep dog, Shetland`, `space bar`, `library`, `paddlewheel, paddle wheel`, `alligator lizard`, `Band Aid`, `Persian cat`, `bull mastiff`, `tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui`, `sports car, sport car`, `football helmet`, `laptop, laptop computer`, `lens cap, lens cover`, `tennis ball`, `violin, fiddle`, `lab coat, laboratory coat`, `cinema, movie theater, movie theatre, movie house, picture palace`, `weasel`, `bow tie, bow-tie, bowtie`, `macaw`, `dough`, `whiskey jug`, `microphone, mike`, `spoonbill`, `bassinet`, `mud turtle`, `velvet`, `warthog`, `plunger, plumber's helper`, `dugong, Dugong dugon`, `honeycomb`, `badger`, `dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk`, `bee`, `doormat, welcome mat`, `fountain pen`, `giant schnauzer`, `assault rifle, assault gun`, `limpkin, Aramus pictus`, `siamang, Hylobates syndactylus, Symphalangus syndactylus`, `albatross, mollymawk`, `confectionery, confectionary, candy store`, `harp`, `parachute, chute`, `barrel, cask`, `tank, army tank, armored combat vehicle, armoured combat vehicle`, `collie`, `kite`, `puck, hockey puck`, `stupa, tope`, `buckeye, horse chestnut, conker`, `patio, terrace`, `broom`, `Dandie Dinmont, Dandie Dinmont terrier`, `scorpion`, `agaric`, `balloon`, `bucket, pail`, `squirrel monkey, Saimiri sciureus`, `Eskimo dog, husky`, `zebra`, `garter snake, grass snake`, `indri, indris, Indri indri, Indri brevicaudatus`, `tractor`, `guinea pig, Cavia cobaya`, `maraca`, `red-backed sandpiper, dunlin, Erolia alpina`, `bullfrog, Rana catesbeiana`, `trilobite`, `Japanese spaniel`, `gorilla, Gorilla gorilla`, `monastery`, `centipede`, `terrapin`, `llama`, `long-horned beetle, longicorn, longicorn beetle`, `boxer`, `curly-coated retriever`, `mortar`, `hammerhead, hammerhead shark`, `goldfinch, Carduelis carduelis`, `garden spider, Aranea diademata`, `stopwatch, stop watch`, `grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus`, `leaf beetle, chrysomelid`, `birdhouse`, `king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica`, `stole`, `bell pepper`, `radiator`, `flatworm, platyhelminth`, `mushroom`, `Scotch terrier, Scottish terrier, Scottie`, `liner, ocean liner`, `toilet seat`, `lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens`, `zucchini, courgette`, `harvestman, daddy longlegs, Phalangium opilio`, `Newfoundland, Newfoundland dog`, `flamingo`, `whiptail, whiptail lizard`, `geyser`, `cleaver, meat cleaver, chopper`, `sea cucumber, holothurian`, `American egret, great white heron, Egretta albus`, `parking meter`, `beacon, lighthouse, beacon light, pharos`, `coucal`, `motor scooter, scooter`, `mitten`, `cannon`, `weevil`, `megalith, megalithic structure`, `stinkhorn, carrion fungus`, `ear, spike, capitulum`, `box turtle, box tortoise`, `snowplow, snowplough`, `tench, Tinca tinca`, `modem`, `tobacco shop, tobacconist shop, tobacconist`, `barn`, `skunk, polecat, wood pussy`, `African grey, African gray, Psittacus erithacus`, `Madagascar cat, ring-tailed lemur, Lemur catta`, `holster`, `barometer`, `sleeping bag`, `washer, automatic washer, washing machine`, `recreational vehicle, RV, R.V.`, `drake`, `tray`, `butcher shop, meat market`, `china cabinet, china closet`, `medicine chest, medicine cabinet`, `photocopier`, `Yorkshire terrier`, `starfish, sea star`, `racket, racquet`, `park bench`, `Labrador retriever`, `whistle`, `clog, geta, patten, sabot`, `volcano`, `quilt, comforter, comfort, puff`, `leopard, Panthera pardus`, `cauliflower`, `swimming trunks, bathing trunks`, `American chameleon, anole, Anolis carolinensis`, `alp`, `mortarboard`, `barracouta, snoek`, `cocker spaniel, English cocker spaniel, cocker`, `space shuttle`, `beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon`, `harmonica, mouth organ, harp, mouth harp`, `gasmask, respirator, gas helmet`, `wombat`, `Model T`, `wild boar, boar, Sus scrofa`, `hermit crab`, `flat-coated retriever`, `rotisserie`, `jinrikisha, ricksha, rickshaw`, `trifle`, `bannister, banister, balustrade, balusters, handrail`, `go-kart`, `bakery, bakeshop, bakehouse`, `ski mask`, `dock, dockage, docking facility`, `Egyptian cat`, `oxcart`, `redbone`, `shoe shop, shoe-shop, shoe store`, `convertible`, `ox`, `crayfish, crawfish, crawdad, crawdaddy`, `cowboy hat, ten-gallon hat`, `conch`, `spaghetti squash`, `toy poodle`, `saltshaker, salt shaker`, `microwave, microwave oven`, `triceratops`, `necklace`, `castle`, `streetcar, tram, tramcar, trolley, trolley car`, `eel`, `diaper, nappy, napkin`, `standard poodle`, `prayer rug, prayer mat`, `radio, wireless`, `crane`, `envelope`, `rule, ruler`, `gar, garfish, garpike, billfish, Lepisosteus osseus`, `spider monkey, Ateles geoffroyi`, `Irish wolfhound`, `German shepherd, German shepherd dog, German police dog, alsatian`, `umbrella`, `sunglass`, `aircraft carrier, carrier, flattop, attack aircraft carrier`, `water buffalo, water ox, Asiatic buffalo, Bubalus bubalis`, `jellyfish`, `groom, bridegroom`, `tree frog, tree-frog`, `steel arch bridge`, `lemon`, `pickup, pickup truck`, `vault`, `groenendael`, `baseball`, `junco, snowbird`, `maillot, tank suit`, `gazelle`, `jack-o'-lantern`, `military uniform`, `slide rule, slipstick`, `wire-haired fox terrier`, `acorn squash`, `electric fan, blower`, `Brittany spaniel`, `chimpanzee, chimp, Pan troglodytes`, `pillow`, `binder, ring-binder`, `schipperke`, `Afghan hound, Afghan`, `plate rack`, `car mirror`, `hand-held computer, hand-held microcomputer`, `papillon`, `schooner`, `Bedlington terrier`, `cellular telephone, cellular phone, cellphone, cell, mobile phone`, `altar`, `Chesapeake Bay retriever`, `cabbage butterfly`, `polecat, fitch, foulmart, foumart, Mustela putorius`, `comic book`, `French horn, horn`, `daisy`, `organ, pipe organ`, `mashed potato`, `acorn`, `fly`, `chain`, `American alligator, Alligator mississipiensis`, `mink`, `garbage truck, dustcart`, `totem pole`, `wine bottle`, `strawberry`, `cricket`, `European fire salamander, Salamandra salamandra`, `coral reef`, `Welsh springer spaniel`, `bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis`, `snorkel`, `bald eagle, American eagle, Haliaeetus leucocephalus`, `meerkat, mierkat`, `grille, radiator grille`, `nematode, nematode worm, roundworm`, `anemone fish`, `corn`, `loggerhead, loggerhead turtle, Caretta caretta`, `palace`, `suit, suit of clothes`, `pineapple, ananas`, `macaque`, `ping-pong ball`, `ram, tup`, `church, church building`, `koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus`, `hare`, `bath towel`, `strainer`, `yawl`, `otterhound, otter hound`, `table lamp`, `king snake, kingsnake`, `lotion`, `lion, king of beasts, Panthera leo`, `thatch, thatched roof`, `basset, basset hound`, `black and gold garden spider, Argiope aurantia`, `barber chair`, `proboscis monkey, Nasalis larvatus`, `consomme`, `Irish terrier`, `Irish water spaniel`, `common iguana, iguana, Iguana iguana`, `Weimaraner`, `Great Dane`, `pug, pug-dog`, `rhinoceros beetle`, `vase`, `brass, memorial tablet, plaque`, `kit fox, Vulpes macrotis`, `king penguin, Aptenodytes patagonica`, `vending machine`, `dalmatian, coach dog, carriage dog`, `rock beauty, Holocanthus tricolor`, `pole`, `cuirass`, `bolete`, `jackfruit, jak, jack`, `monarch, monarch butterfly, milkweed butterfly, Danaus plexippus`, `chow, chow chow`, `nail`, `packet`, `half track`, `Lhasa, Lhasa apso`, `boathouse`, `hay`, `valley, vale`, `slot, one-armed bandit`, `volleyball`, `carton`, `shower cap`, `tile roof`, `lacewing, lacewing fly`, `patas, hussar monkey, Erythrocebus patas`, `boa constrictor, Constrictor constrictor`, `black swan, Cygnus atratus`, `lampshade, lamp shade`, `mousetrap`, `crutch`, `vestment`, `Pekinese, Pekingese, Peke`, `tusker`, `warplane, military plane`, `sandal`, `screw`, `custard apple`, `Scottish deerhound, deerhound`, `spindle`, `keeshond`, `hummingbird`, `letter opener, paper knife, paperknife`, `cucumber, cuke`, `bearskin, busby, shako`, `fire engine, fire truck`, `trombone`, `ringneck snake, ring-necked snake, ring snake`, `sorrel`, `fire screen, fireguard`, `paper towel`, `flute, transverse flute`, `hotdog, hot dog, red hot`, `Indian elephant, Elephas maximus`, `mosque`, `stingray`, `Rhodesian ridgeback`, `four-poster` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_demo_en_4.1.0_3.0_1660167069440.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_demo_en_4.1.0_3.0_1660167069440.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_demo", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_demo", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_demo| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|324.8 MB| --- layout: model title: Word2Vec Embeddings in Lombard (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, lmo, open_source] task: Embeddings language: lmo edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_lmo_3.4.1_3.0_1647443293397.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_lmo_3.4.1_3.0_1647443293397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_diseases", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive. ']], ["text"])) ``` ```scala ... val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_diseases", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive. ").toDF("text") val result = pipeline.fit(data).transform(data) } ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_deepset_bert_base_uncased_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_deepset_bert_base_uncased_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_deepset_bert_base_uncased_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/deepset/bert-base-uncased-squad2 - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - https://twitter.com/deepset_ai - http://www.deepset.ai/jobs - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/ - https://deepset.ai/german-bert - https://www.linkedin.com/company/deepset-ai/ - https://github.com/deepset-ai/FARM - https://deepset.ai/germanquad --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-256-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8_en_4.0.0_3.0_1657184863712.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8_en_4.0.0_3.0_1657184863712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-256-finetuned-squad-seed-8 --- layout: model title: Legal Forbearance Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_forbearance_agreement_bert date: 2023-02-02 tags: [en, legal, classification, forbearance, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_forbearance_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `forbearance-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `forbearance-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_forbearance_agreement_bert_en_1.0.0_3.0_1675359983427.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_forbearance_agreement_bert_en_1.0.0_3.0_1675359983427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_forbearance_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[forbearance-agreement]| |[other]| |[other]| |[forbearance-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_forbearance_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support forbearance-agreement 0.97 1.00 0.99 37 other 1.00 0.99 0.99 73 accuracy - - 0.99 110 macro-avg 0.99 0.99 0.99 110 weighted-avg 0.99 0.99 0.99 110 ``` --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_fpdm_hier_roberta_FT_newsqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fpdm_hier_roberta_FT_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_hier_roberta_FT_newsqa_en_4.0.0_3.0_1655728663969.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_hier_roberta_FT_newsqa_en_4.0.0_3.0_1655728663969.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_hier_roberta_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_fpdm_hier_roberta_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.roberta.qa_fpdm_hier_roberta_ft_newsqa.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fpdm_hier_roberta_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|458.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/fpdm_hier_roberta_FT_newsqa --- layout: model title: Hindi BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_dev date: 2022-07-07 tags: [hi, open_source, bert, question_answering] task: Question Answering language: hi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-hi` is a Hindi model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_dev_hi_4.0.0_3.0_1657190202881.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_dev_hi_4.0.0_3.0_1657190202881.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_dev","hi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["मेरा नाम क्या है?", "मेरा नाम क्लारा है और मैं बर्कले में रहता हूं।"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_dev","hi") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("मेरा नाम क्या है?", "मेरा नाम क्लारा है और मैं बर्कले में रहता हूं।").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|hi| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-hi --- layout: model title: Fast Neural Machine Translation Model from English to Philippine Languages author: John Snow Labs name: opus_mt_en_phi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, phi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `phi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_phi_xx_2.7.0_2.4_1609169890617.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_phi_xx_2.7.0_2.4_1609169890617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_phi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Danish Legal Roberta Embeddings author: John Snow Labs name: roberta_base_danish_legal date: 2023-02-16 tags: [da, danish, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: da edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-danish-roberta-base` is a Danish model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_danish_legal_da_4.2.4_3.0_1676576630307.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_danish_legal_da_4.2.4_3.0_1676576630307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_danish_legal", "da")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_danish_legal", "da") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_danish_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|da| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-danish-roberta-base --- layout: model title: BERT Sequence Classification - Classify into News Categories author: John Snow Labs name: bert_sequence_classifier_age_news date: 2021-11-07 tags: [news, classification, bert_for_sequence_classification, en, open_source, ag_news] task: Text Classification language: en nav_key: models edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is imported from `Hugging Face-models`. It is a BERT-Mini fine-tuned version of the `age_news` dataset. ## Predicted Entities `World`, `Sports`, `Business`, `Sci/Tech` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_age_news_en_3.3.2_2.4_1636288849469.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_age_news_en_3.3.2_2.4_1636288849469.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['Sci/Tech'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_age_news| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[label]| |Language:|en| |Case sensitive:|true| ## Data Source [https://huggingface.co/mrm8488/bert-mini-finetuned-age_news-classification](https://huggingface.co/mrm8488/bert-mini-finetuned-age_news-classification) ## Benchmarking ```bash Test set accuracy: 0.93 ``` --- layout: model title: Extract relations between problem, treatment and test entities (ReDL) author: John Snow Labs name: redl_clinical_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations like `TrIP` : a certain treatment has improved a medical problem and 7 other such relations between problem, treatment and test entities. ## Predicted Entities `PIP`, `TeCP`, `TeRP`, `TrAP`, `TrCP`, `TrIP`, `TrNAP`, `TrWP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_clinical_biobert_en_2.7.3_2.4_1612443963755.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_clinical_biobert_en_2.7.3_2.4_1612443963755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|:-----------|:----------|----------------:|--------------:|:--------------------------------------|:----------|----------------:|--------------:|:-------------------------|-------------:| | 0 | PIP | PROBLEM | 39 | 67 | gestational diabetes mellitus | PROBLEM | 157 | 160 | T2DM | 0.763447 | | 1 | PIP | PROBLEM | 39 | 67 | gestational diabetes mellitus | PROBLEM | 289 | 295 | obesity | 0.682246 | | 2 | PIP | PROBLEM | 117 | 153 | subsequent type two diabetes mellitus | PROBLEM | 187 | 210 | HTG-induced pancreatitis | 0.610396 | | 3 | PIP | PROBLEM | 117 | 153 | subsequent type two diabetes mellitus | PROBLEM | 264 | 281 | an acute hepatitis | 0.726894 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_clinical_biobert| |Compatibility:|Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Data Source Trained on 2010 i2b2 relation challenge. ## Benchmarking ```bash Relation Recall Precision F1 Support PIP 0.859 0.878 0.869 1435 TeCP 0.629 0.782 0.697 337 TeRP 0.903 0.929 0.916 2034 TrAP 0.872 0.866 0.869 1693 TrCP 0.641 0.677 0.659 340 TrIP 0.517 0.796 0.627 151 TrNAP 0.402 0.672 0.503 112 TrWP 0.257 0.824 0.392 109 Avg. 0.635 0.803 0.691 ``` --- layout: model title: Sentence Entity Resolver for ICD10-CM (general 3 character codes) author: John Snow Labs name: sbiobertresolve_icd10cm_generalised date: 2021-09-29 tags: [licensed, clinical, en, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.2.1 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It predicts ICD codes up to 3 characters (according to ICD10 code structure the first three characters represent general type of the injury or disease). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_generalised_en_3.2.1_3.0_1632938859569.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_generalised_en_3.2.1_3.0_1632938859569.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_icd10cm_generalised``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. ```PROBLEM``` set in ```.setWhiteList()```.

## Results ```bash | | chunk | begin | end | entity | code | code_desc | distance | all_k_resolutions | all_k_codes | |---:|:----------------------------|--------:|------:|:---------|:-------|:---------------------------------------------------------|-----------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------| | 0 | hypertension | 68 | 79 | PROBLEM | I10 | hypertension | 0 | hypertension:::hypertension (high blood pressure):::h/o: hypertension:::fh: hypertension:::hypertensive heart disease:::labile hypertension:::history of hypertension (situation):::endocrine hypertension | I10:::I15:::Z86:::Z82:::I11:::R03:::Z87:::E27 | | 1 | chronic renal insufficiency | 83 | 109 | PROBLEM | N18 | chronic renal impairment | 0.014 | chronic renal impairment:::renal insufficiency:::renal failure:::anaemia of chronic renal insufficiency:::impaired renal function disorder:::history of renal insufficiency:::prerenal renal failure:::abnormal renal function:::abnormal renal function | N18:::P96:::N19:::D63:::N28:::Z87:::N17:::N25:::R94 | | 2 | COPD | 113 | 116 | PROBLEM | J44 | chronic obstructive lung disease (disorder) | 0.1197 | chronic obstructive lung disease (disorder):::chronic obstructive pulmonary disease leaflet given:::chronic pulmonary congestion (disorder):::chronic respiratory failure (disorder):::chronic respiratory insufficiency:::cor pulmonale (chronic):::history of - chronic lung disease (situation) | J44:::Z76:::J81:::J96:::R06:::I27:::Z87 | | 3 | gastritis | 120 | 128 | PROBLEM | K29 | gastritis | 0 | gastritis:::bacterial gastritis:::parasitic gastritis | K29:::B96:::K93 | | 4 | TIA | 136 | 138 | PROBLEM | S06 | cerebral concussion | 0.1662 | cerebral concussion:::transient ischemic attack (disorder):::thalamic stroke:::cerebral trauma:::stroke:::traumatic amputation:::spinal cord stroke | S06:::G45:::I63:::S09:::I64:::T14:::G95 | | 5 | a non-ST elevation MI | 182 | 202 | PROBLEM | I21 | non-st elevation (nstemi) myocardial infarction | 0.1615 | non-st elevation (nstemi) myocardial infarction:::nonruptured cerebral artery dissection:::acute stroke, nonatherosclerotic:::nontraumatic ischemic infarction of muscle, unsp shoulder:::history of nonatherosclerotic stroke without residual deficits:::non-traumatic cerebral hemorrhage | I21:::I67:::I63:::M62:::Z86:::I61 | | 6 | Guaiac positive stools | 208 | 229 | PROBLEM | R85 | abnormal anal pap | 0.1807 | abnormal anal pap:::straining at stool (finding):::amine test positive:::appendiceal colic:::fecal smearing:::epiploic appendagitis:::diverticulosis of intestine (finding):::appendicitis (disorder):::colostomy present (finding):::thickened anal verge (finding):::anal fissure:::amoebic enteritis:::zenkers diverticulum | R85:::R19:::Z78:::K38:::R15:::K65:::K57:::K37:::Z93:::K62:::K60:::A06:::K22 | | 7 | mid LAD lesion | 332 | 345 | PROBLEM | I21 | stemi involving left anterior descending coronary artery | 0.1595 | stemi involving left anterior descending coronary artery:::divided left atrium:::disorder of left atrium:::double inlet left ventricle:::left os acromiale:::furuncle of left upper limb:::left anterior fascicular hemiblock (heart rhythm):::aberrant origin of left subclavian artery:::stent in circumflex branch of left coronary artery (finding) | I21:::Q24:::I51:::Q20:::M89:::L02:::I44:::Q27:::Z95 | | 8 | hypotension | 362 | 372 | PROBLEM | I95 | hypotension | 0 | hypotension:::supine hypotensive syndrome | I95:::O26 | | 9 | bradycardia | 378 | 388 | PROBLEM | R00 | bradycardia | 0 | bradycardia:::bradycardia (finding):::drug-induced bradycardia:::bradycardia (disorder) | R00:::P29:::T50:::P20 | | 10 | vagal reaction | 466 | 479 | PROBLEM | G52 | vagus nerve finding | 0.0926 | vagus nerve finding:::vasomotor reaction:::vesicular breathing (finding):::abdominal muscle tone - finding:::agonizing state:::paresthesia (finding):::glossolalia (finding):::tactile alteration (finding) | G52:::I73:::R09:::R19:::R45:::R20:::R41:::R44 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_generalised| |Compatibility:|Healthcare NLP 3.2.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on ICD10 Clinical Modification dataset with `sbiobert_base_cased_mli` sentence embeddings. https://www.icd10data.com/ICD10CM/Codes/ --- layout: model title: Pretrained Pipeline for Few-NERD-General NER Model author: John Snow Labs name: nerdl_fewnerd_100d_pipeline date: 2021-12-03 tags: [fewnerd, nerdl, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.3.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on Few-NERD model and it detects : `PERSON`, `ORGANIZATION`, `LOCATION`, `ART`, `BUILDING`, `PRODUCT`, `EVENT`, `OTHER` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_pipeline_en_3.3.1_2.4_1638523061152.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_pipeline_en_3.3.1_2.4_1638523061152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------+------------+ |chunk |ner_label | +-----------------------+------------+ |Kentucky Fried Chicken |ORGANIZATION| |Anglo-Egyptian War |EVENT | |battle of Tell El Kebir|EVENT | |Egypt Medal |OTHER | |Order of Medjidie |OTHER | +-----------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_fewnerd_100d_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.1+| |License:|Open Source| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter - Finisher --- layout: model title: English asr_wav2vec2_large_xls_r_300m_cantonese TFWav2Vec2ForCTC from ivanlau author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_cantonese date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_cantonese` is a English model originally trained by ivanlau. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_cantonese_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_cantonese_en_4.2.0_3.0_1664113131550.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_cantonese_en_4.2.0_3.0_1664113131550.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_cantonese', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_cantonese", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_cantonese| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Stopwords Remover for Hungarian language (219 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, hu, open_source] task: Stop Words Removal language: hu edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_hu_3.4.1_3.0_1646673036995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_hu_3.4.1_3.0_1646673036995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+ |result | +---------+ |[jobb, ,]| +---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|hu| |Size:|2.1 KB| --- layout: model title: English DistilBertForQuestionAnswering Cased model (from jteng) author: John Snow Labs name: distilbert_qa_finetuned_syllabus date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-syllabus` is a English model originally trained by `jteng`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_finetuned_syllabus_en_4.3.0_3.0_1672765843797.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_finetuned_syllabus_en_4.3.0_3.0_1672765843797.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_finetuned_syllabus","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_finetuned_syllabus","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_finetuned_syllabus| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jteng/bert-finetuned-syllabus --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from ardallie) author: John Snow Labs name: xlmroberta_ner_ardallie_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `ardallie`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_ardallie_base_finetuned_panx_de_4.1.0_3.0_1660430915952.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_ardallie_base_finetuned_panx_de_4.1.0_3.0_1660430915952.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_ardallie_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_ardallie_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_ardallie_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ardallie/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English T5ForConditionalGeneration Cased model (from pitehu) author: John Snow Labs name: t5_ner_conll_entityreplace date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `T5_NER_CONLL_ENTITYREPLACE` is a English model originally trained by `pitehu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_ner_conll_entityreplace_en_4.3.0_3.0_1675099568513.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_ner_conll_entityreplace_en_4.3.0_3.0_1675099568513.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ner_conll_entityreplace","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ner_conll_entityreplace","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_ner_conll_entityreplace| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|275.5 MB| ## References - https://huggingface.co/pitehu/T5_NER_CONLL_ENTITYREPLACE - https://arxiv.org/pdf/2111.10952.pdf - https://arxiv.org/pdf/1810.04805.pdf --- layout: model title: Legal Dispute resolution Clause Binary Classifier author: John Snow Labs name: legclf_dispute_resolution_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `dispute-resolution` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `dispute-resolution` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_dispute_resolution_clause_en_1.0.0_3.2_1660122359464.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_dispute_resolution_clause_en_1.0.0_3.2_1660122359464.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_dispute_resolution_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[dispute-resolution]| |[other]| |[other]| |[dispute-resolution]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_dispute_resolution_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support dispute-resolution 0.84 0.84 0.84 32 other 0.94 0.94 0.94 84 accuracy - - 0.91 116 macro-avg 0.89 0.89 0.89 116 weighted-avg 0.91 0.91 0.91 116 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from allenai) author: John Snow Labs name: t5_small_next_word_generator_qoogle date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-next-word-generator-qoogle` is a English model originally trained by `allenai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_next_word_generator_qoogle_en_4.3.0_3.0_1675126551905.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_next_word_generator_qoogle_en_4.3.0_3.0_1675126551905.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_next_word_generator_qoogle","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_next_word_generator_qoogle","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_next_word_generator_qoogle| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|148.1 MB| ## References - https://huggingface.co/allenai/t5-small-next-word-generator-qoogle --- layout: model title: Translate English to Tagalog Pipeline author: John Snow Labs name: translate_en_tl date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tl, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tl_xx_2.7.0_2.4_1609691452242.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tl_xx_2.7.0_2.4_1609691452242.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tl').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tl| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_name_vsv_all_901529445 date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-name_vsv_all-901529445` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `OOV`, `Name` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1677881751372.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1677881751372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_name_vsv_all_901529445| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-name_vsv_all-901529445 --- layout: model title: BioBERT Sentence Embeddings (PMC) author: John Snow Labs name: sent_biobert_pmc_base_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_pmc_base_cased_en_2.6.2_2.4_1600532770743.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_pmc_base_cased_en_2.6.2_2.4_1600532770743.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pmc_base_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pmc_base_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.biobert.pmc_base_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_pmc_base_cased_embeddings I hate cancer [0.34035101532936096, 0.04413360357284546, -0.... Antibiotics aren't painkiller [0.4397204518318176, 0.066007100045681, -0.114... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.2| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: BERT Embeddings trained on MEDLINE/PubMed and fine-tuned on SQuAD 2.0 author: John Snow Labs name: bert_pubmed_squad2 date: 2021-08-30 tags: [en, open_source, squad_2_dataset, medline_pubmed_dataset, bert_embeddings] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/pubmed/1 and fine-tuned on SQuAD 2.0. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pubmed_squad2_en_3.2.0_3.0_1630323544592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pubmed_squad2_en_3.2.0_3.0_1630323544592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_pubmed_squad2", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_pubmed_squad2", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.pubmed_squad2').predict(text, output_level='token') embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pubmed_squad2| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [Wikipedia dataset](https://dumps.wikimedia.org/) [2]: [BooksCorpus dataset](http://yknzhu.wixsite.com/mbweb) [3]: [Stanford Queston Answering (SQuAD 2.0) dataset](https://rajpurkar.github.io/SQuAD-explorer/) [4]: [MEDLINE/PubMed dataset](https://www.nlm.nih.gov/databases/download/pubmed_medline.html) This Model has been imported from: https://tfhub.dev/google/experts/bert/pubmed/squad2/2 --- layout: model title: Bashkir asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt TFWav2Vec2ForCTC from AigizK author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt date: 2022-09-24 tags: [wav2vec2, ba, audio, open_source, asr] task: Automatic Speech Recognition language: ba edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt` is a Bashkir model originally trained by AigizK. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_ba_4.2.0_3.0_1664040309179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_ba_4.2.0_3.0_1664040309179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt", "ba")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt", "ba") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ba| |Size:|1.2 GB| --- layout: model title: Hindi BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_hi_cased date: 2022-12-02 tags: [hi, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: hi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-hi-cased` is a Hindi model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_hi_cased_hi_4.2.4_3.0_1670017763072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_hi_cased_hi_4.2.4_3.0_1670017763072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_hi_cased","hi") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_hi_cased","hi") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_hi_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|339.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-hi-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: German BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_de_cased date: 2022-12-02 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-de-cased` is a German model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_de_cased_de_4.2.4_3.0_1670016505471.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_de_cased_de_4.2.4_3.0_1670016505471.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_de_cased","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_de_cased","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_de_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|398.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-de-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate English to Twi Pipeline author: John Snow Labs name: translate_en_tw date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tw, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tw_xx_2.7.0_2.4_1609691518744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tw_xx_2.7.0_2.4_1609691518744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tw').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tw| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Marscen) author: John Snow Labs name: distilbert_qa_marscen_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Marscen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_marscen_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768784222.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_marscen_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768784222.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_marscen_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_marscen_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_marscen_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Marscen/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering Cased model (from spasis) author: John Snow Labs name: bert_qa_spasis_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `spasis`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spasis_finetuned_squad_en_4.0.0_3.0_1657186715488.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spasis_finetuned_squad_en_4.0.0_3.0_1657186715488.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spasis_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spasis_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spasis_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/spasis/bert-finetuned-squad --- layout: model title: Extract Test Entities from Voice of the Patient Documents (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_test_emb_clinical_medium date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, test] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts test mentions from the documents transferred from the patient’s own sentences. ## Predicted Entities `VitalTest`, `Test`, `Measurements`, `TestResult` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_emb_clinical_medium_en_4.4.3_3.0_1686076924102.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_emb_clinical_medium_en_4.4.3_3.0_1686076924102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------|:------------| | thyroid levels | Test | | blood test | Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_test_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_medium| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 VitalTest 162 32 10 172 0.84 0.94 0.89 Test 1040 118 168 1208 0.90 0.86 0.88 Measurements 136 22 50 186 0.86 0.73 0.79 TestResult 360 109 164 524 0.77 0.69 0.73 macro_avg 1698 281 392 2090 0.84 0.80 0.82 micro_avg 1698 281 392 2090 0.86 0.81 0.84 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from ydshieh) author: John Snow Labs name: roberta_qa_ydshieh_base_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `ydshieh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ydshieh_base_squad2_en_4.2.4_3.0_1669986831741.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ydshieh_base_squad2_en_4.2.4_3.0_1669986831741.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ydshieh_base_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ydshieh_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ydshieh_base_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ydshieh/roberta-base-squad2 - https://github.com/deepset-ai/FARM/issues/552 - https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py - https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://github.com/deepset-ai/haystack/ - https://workablehr.s3.amazonaws.com/uploads/account/logo/476306/logo - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Pipeline to Extract the Names of Drugs & Chemicals author: John Snow Labs name: ner_chemd_clinical_pipeline date: 2023-03-14 tags: [chemdner, chemd, ner, clinical, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chemd_clinical](https://nlp.johnsnowlabs.com/2021/11/04/ner_chemd_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemd_clinical_pipeline_en_4.3.0_3.2_1678778578175.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemd_clinical_pipeline_en_4.3.0_3.2_1678778578175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_chemd_clinical_pipeline", "en", "clinical/models") text = '''Isolation, Structure Elucidation, and Iron-Binding Properties of Lystabactins, Siderophores Isolated from a Marine Pseudoalteromonas sp. The marine bacterium Pseudoalteromonas sp. S2B, isolated from the Gulf of Mexico after the Deepwater Horizon oil spill, was found to produce lystabactins A, B, and C (1-3), three new siderophores. The structures were elucidated through mass spectrometry, amino acid analysis, and NMR. The lystabactins are composed of serine (Ser), asparagine (Asn), two formylated/hydroxylated ornithines (FOHOrn), dihydroxy benzoic acid (Dhb), and a very unusual nonproteinogenic amino acid, 4,8-diamino-3-hydroxyoctanoic acid (LySta). The iron-binding properties of the compounds were investigated through a spectrophotometric competition.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_chemd_clinical_pipeline", "en", "clinical/models") val text = "Isolation, Structure Elucidation, and Iron-Binding Properties of Lystabactins, Siderophores Isolated from a Marine Pseudoalteromonas sp. The marine bacterium Pseudoalteromonas sp. S2B, isolated from the Gulf of Mexico after the Deepwater Horizon oil spill, was found to produce lystabactins A, B, and C (1-3), three new siderophores. The structures were elucidated through mass spectrometry, amino acid analysis, and NMR. The lystabactins are composed of serine (Ser), asparagine (Asn), two formylated/hydroxylated ornithines (FOHOrn), dihydroxy benzoic acid (Dhb), and a very unusual nonproteinogenic amino acid, 4,8-diamino-3-hydroxyoctanoic acid (LySta). The iron-binding properties of the compounds were investigated through a spectrophotometric competition." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------------------------|--------:|------:|:-------------|-------------:| | 0 | Lystabactins | 65 | 76 | FAMILY | 0.9841 | | 1 | lystabactins A, B, and C | 278 | 301 | MULTIPLE | 0.813429 | | 2 | amino acid | 392 | 401 | FAMILY | 0.74585 | | 3 | lystabactins | 426 | 437 | FAMILY | 0.8007 | | 4 | serine | 455 | 460 | TRIVIAL | 0.9924 | | 5 | Ser | 463 | 465 | FORMULA | 0.9999 | | 6 | asparagine | 469 | 478 | TRIVIAL | 0.9795 | | 7 | Asn | 481 | 483 | FORMULA | 0.9999 | | 8 | formylated/hydroxylated ornithines | 491 | 524 | FAMILY | 0.50085 | | 9 | FOHOrn | 527 | 532 | FORMULA | 0.509 | | 10 | dihydroxy benzoic acid | 536 | 557 | SYSTEMATIC | 0.6346 | | 11 | amino acid | 602 | 611 | FAMILY | 0.4204 | | 12 | 4,8-diamino-3-hydroxyoctanoic acid | 614 | 647 | SYSTEMATIC | 0.9124 | | 13 | LySta | 650 | 654 | ABBREVIATION | 0.9193 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemd_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Stop Words Cleaner for Yoruba author: John Snow Labs name: stopwords_yo date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: yo edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, yo] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_yo_yo_2.5.4_2.4_1594742440695.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_yo_yo_2.5.4_2.4_1594742440695.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_yo", "yo") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_yo", "yo") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera."""] stopword_df = nlu.load('yo.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=3, result='Yato', metadata={'sentence': '0'}), Row(annotatorType='token', begin=5, end=6, result='si', metadata={'sentence': '0'}), Row(annotatorType='token', begin=8, end=11, result='jijẹ', metadata={'sentence': '0'}), Row(annotatorType='token', begin=13, end=15, result='ọba', metadata={'sentence': '0'}), Row(annotatorType='token', begin=17, end=21, result='ariwa', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_yo| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|yo| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English BertForMaskedLM Base Uncased model (from mlcorelib) author: John Snow Labs name: bert_embeddings_deberta_base_uncased date: 2022-12-06 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deberta-base-uncased` is a English model originally trained by `mlcorelib`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_deberta_base_uncased_en_4.2.4_3.0_1670326237283.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_deberta_base_uncased_en_4.2.4_3.0_1670326237283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_deberta_base_uncased","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_deberta_base_uncased","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_deberta_base_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/mlcorelib/deberta-base-uncased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_squad_2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad-2` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_squad_2_en_4.0.0_3.0_1655734456241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_squad_2_en_4.0.0_3.0_1655734456241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_squad_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_squad_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_v2.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_squad_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|437.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-squad-2 --- layout: model title: Finnish asr_wav2vec2_large_xlsr_finnish TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_finnish` is a Finnish model originally trained by birgermoell. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_finnish_fi_4.2.0_3.0_1664021434692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_finnish_fi_4.2.0_3.0_1664021434692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_finnish', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_finnish", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_finnish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_8_h_768 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-8_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_768_zh_4.2.4_3.0_1670326074741.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_768_zh_4.2.4_3.0_1670326074741.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_8_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|277.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-8_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Detect Clinical Events (Admissions) author: John Snow Labs name: ner_events_admission_clinical date: 2021-03-01 tags: [ner, licensed, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.7.4 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect clinical events in medical text, with a focus on admission entities. ## Predicted Entities `DATE`, `TIME`, `PROBLEM`, `TEST`, `TREATMENT`, `OCCURENCE`, `CLINICAL_DEPT`, `EVIDENTIAL`, `DURATION`, `FREQUENCY`, `ADMISSION`, `DISCHARGE`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_admission_clinical_en_2.7.4_2.4_1614582648104.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_admission_clinical_en_2.7.4_2.4_1614582648104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_events_admission_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient presented to the emergency room last evening"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_events_admission_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The patient presented to the emergency room last evening""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.admission_events").predict("""The patient presented to the emergency room last evening""") ```

## Results ```bash +----+-----------------------------+---------+---------+-----------------+ | | chunk | begin | end | entity | +====+=============================+=========+=========+=================+ | 0 | presented | 12 | 20 | EVIDENTIAL | +----+-----------------------------+---------+---------+-----------------+ | 1 | the emergency room | 25 | 42 | CLINICAL_DEPT | +----+-----------------------------+---------+---------+-----------------+ | 2 | last evening | 44 | 55 | DATE | +----+-----------------------------+---------+---------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_admission_clinical| |Type:|ner| |Compatibility:|Healthcare NLP 2.7.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on augmented/enriched i2b2 events data with clinical_embeddings. The data for Admissions has been enriched specifically. ## Benchmarking ```bash label tp fp fn prec rec f1 I-TIME 42 6 9 0.875 0.8235294 0.8484849 I-TREATMENT 1134 111 312 0.9108434 0.7842324 0.8428094 B-OCCURRENCE 406 344 382 0.5413333 0.51522845 0.52795845 I-DURATION 160 42 71 0.7920792 0.6926407 0.73903 B-DATE 500 32 49 0.9398496 0.9107468 0.92506933 I-DATE 309 54 49 0.8512397 0.8631285 0.8571429 B-ADMISSION 206 1 2 0.9951691 0.99038464 0.9927711 I-PROBLEM 2394 390 412 0.85991377 0.85317177 0.8565295 B-CLINICAL_DEPT 327 64 77 0.8363171 0.8094059 0.8226415 B-TIME 44 12 15 0.78571427 0.7457627 0.76521736 I-CLINICAL_DEPT 597 62 78 0.90591806 0.8844444 0.8950525 B-PROBLEM 1643 260 252 0.86337364 0.86701846 0.86519223 I-FREQUENCY 35 21 39 0.625 0.47297296 0.5384615 I-TEST 1082 171 117 0.86352754 0.9024187 0.8825449 B-TEST 781 125 127 0.8620309 0.86013216 0.86108047 B-TREATMENT 1283 176 202 0.87936944 0.8639731 0.87160325 B-DISCHARGE 155 0 1 1.0 0.99358976 0.99678457 B-EVIDENTIAL 269 25 75 0.914966 0.78197676 0.84326017 B-DURATION 97 43 44 0.69285715 0.6879433 0.6903914 B-FREQUENCY 70 16 33 0.81395346 0.6796116 0.7407407 tp: 11841 fp: 2366 fn: 2680 labels: 22 Macro-average prec: 0.8137135, rec: 0.7533389, f1: 0.7823631 Micro-average prec: 0.83346236, rec: 0.8154397, f1: 0.8243525 ``` --- layout: model title: Legal Whereas Clause Binary Classifier author: John Snow Labs name: legclf_cuad_whereas_clause date: 2022-11-25 tags: [whereas, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `whereas` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `whereas`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_whereas_clause_en_1.0.0_3.0_1669379828062.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_whereas_clause_en_1.0.0_3.0_1669379828062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_cuad_whereas_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[whereas]| |[other]| |[other]| |[whereas]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_whereas_clause| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.4 MB| ## References In-house annotations on CUAD dataset ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.94 0.96 67 whereas 0.91 0.98 0.94 41 accuracy - - 0.95 108 macro-avg 0.95 0.96 0.95 108 weighted-avg 0.96 0.95 0.95 108 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from T-qualizer) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_advers date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-advers` is a English model originally trained by `T-qualizer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_advers_en_4.0.0_3.0_1654723842338.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_advers_en_4.0.0_3.0_1654723842338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_advers","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_advers","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_advers| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/T-qualizer/distilbert-base-uncased-finetuned-advers --- layout: model title: Applicable Law Clause NER Model author: John Snow Labs name: legner_applicable_law_clause date: 2023-01-12 tags: [en, ner, licensed, applicable_law] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model aimed to be used in `applicable_law` clauses to retrieve entities as `APPLIC_LAW`. Make sure you run this model only on `applicable_law` clauses after you filter them using `legclf_applicable_law_cuad` model. ## Predicted Entities `APPLIC_LAW` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_applicable_law_clause_en_1.0.0_3.0_1673558480167.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_applicable_law_clause_en_1.0.0_3.0_1673558480167.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------+----------+----------+ |chunk |ner_label |confidence| +----------------------------------------+----------+----------+ |laws of the Province of British Columbia|APPLIC_LAW|0.95625716| +----------------------------------------+----------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_applicable_law_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.1 MB| ## References In-house dataset ## Benchmarking ```bash label precision recall f1-score support B-APPLIC_LAW 0.90 0.89 0.90 84 I-APPLIC_LAW 0.98 0.93 0.96 425 micro-avg 0.97 0.93 0.95 509 macro-avg 0.94 0.91 0.93 509 weighted-avg 0.97 0.93 0.95 509 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from laampt) author: John Snow Labs name: distilbert_qa_laampt_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `laampt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_laampt_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771910439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_laampt_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771910439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_laampt_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_laampt_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_laampt_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/laampt/distilbert-base-uncased-finetuned-squad --- layout: model title: German asr_wav2vec2_base_german TFWav2Vec2ForCTC from aware-ai author: John Snow Labs name: pipeline_asr_wav2vec2_base_german date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_german` is a German model originally trained by aware-ai. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_german_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_german_de_4.2.0_3.0_1664099298596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_german_de_4.2.0_3.0_1664099298596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_german', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_german", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_german| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from NeuML) author: John Snow Labs name: bert_qa_bert_small_cord19_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-small-cord19-squad2` is a English model orginally trained by `NeuML`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_cord19_squad2_en_4.0.0_3.0_1654184738698.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_cord19_squad2_en_4.0.0_3.0_1654184738698.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_small_cord19_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_small_cord19_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_cord19.bert.small").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_small_cord19_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|130.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/NeuML/bert-small-cord19-squad2 --- layout: model title: Spanish Bert Embeddings (from amine) author: John Snow Labs name: bert_embeddings_bert_base_5lang_cased date: 2022-04-11 tags: [bert, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-5lang-cased` is a Spanish model orginally trained by `amine`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_5lang_cased_es_3.4.2_3.0_1649671304061.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_5lang_cased_es_3.4.2_3.0_1649671304061.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_5lang_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|464.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/amine/bert-base-5lang-cased - https://cloud.google.com/compute/docs/machine-types#n1_machine_type --- layout: model title: English DistilBertForQuestionAnswering Cased model (from saraks) author: John Snow Labs name: distilbert_qa_cuad_document_name_08_25 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cuad-distil-document_name-08-25` is a English model originally trained by `saraks`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_document_name_08_25_en_4.3.0_3.0_1672766062646.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_document_name_08_25_en_4.3.0_3.0_1672766062646.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_document_name_08_25","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_document_name_08_25","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cuad_document_name_08_25| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saraks/cuad-distil-document_name-08-25 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from srmukundb) author: John Snow Labs name: distilbert_qa_srmukundb_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `srmukundb`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_srmukundb_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772870213.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_srmukundb_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772870213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_srmukundb_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_srmukundb_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_srmukundb_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/srmukundb/distilbert-base-uncased-finetuned-squad --- layout: model title: English image_classifier_vit_asl ViTForImageClassification from akahana author: John Snow Labs name: image_classifier_vit_asl date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_asl` is a English model originally trained by akahana. ## Predicted Entities `E`, `del`, `X`, `N`, `T`, `Y`, `J`, `U`, `F`, `A`, `M`, `I`, `G`, `nothing`, `V`, `Q`, `L`, `space`, `B`, `P`, `C`, `H`, `W`, `K`, `R`, `O`, `D`, `Z`, `S` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_asl_en_4.1.0_3.0_1660166442859.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_asl_en_4.1.0_3.0_1660166442859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_asl", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_asl", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_asl| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.0 MB| --- layout: model title: Detect Assertion Status (assertion_dl_healthcare) author: John Snow Labs name: assertion_dl_healthcare class: AssertionDLModel reference embedding: healthcare_embeddings language: en nav_key: models repository: clinical/models date: 2020-09-23 task: Assertion Status edition: Healthcare NLP 2.6.0 spark_version: 2.4 tags: [clinical,licensed,assertion,en] supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Assertion of Clinical Entities based on Deep Learning. ## Predicted Entities `hypothetical`, `present`, `absent`, `possible`, `conditional`, `associated_with_someone_else`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_healthcare_en_2.6.0_2.4_1600849811713.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_healthcare_en_2.6.0_2.4_1600849811713.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel, AssertionDLModel.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("assertion_dl_healthcare","en","clinical/models")\ .setInputCols(["document","ner_chunk","embeddings"])\ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion]) model = nlpPipeline.fit(spark.createDataFrame([['Patient has a headache for the last 2 weeks and appears anxious when she walks fast. No alopecia noted. She denies pain']]).toDF("text")) results = model.transform(data) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("assertion_dl_healthcare","en","clinical/models") .setInputCols("document","ner_chunk","embeddings") .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("Patient has a headache for the last 2 weeks and appears anxious when she walks fast. No alopecia noted. She denies pain").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.assert.healthcare").predict("""Patient has a headache for the last 2 weeks and appears anxious when she walks fast. No alopecia noted. She denies pain""") ```

{:.h2_title} ## Result ```bash | | chunks | entities| assertion | |--:|-----------:|--------:|------------:| | 0 | a headache | PROBLEM | present | | 1 | anxious | PROBLEM | conditional | | 2 | alopecia | PROBLEM | absent | | 3 | pain | PROBLEM | absent | ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------------| | Name: | assertion_dl_healthcare | | Type: | AssertionDLModel | | Compatibility: | 2.6.0 | | License: | Licensed | |Edition:|Official| | |Input labels: | [document, chunk, word_embeddings] | |Output labels: | [assertion] | | Language: | en | | Case sensitive: | False | | Dependencies: | embeddings_healthcare_100d | {:.h2_title} ## Data Source Trained using ``embeddings_clinical`` on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text from https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ {:.h2_title} ## Benchmarking ```bash label prec rec f1 absent 0.9289 0.9466 0.9377 present 0.9433 0.9559 0.9496 conditional 0.6888 0.5 0.5794 associated_with_someone_else 0.9285 0.9122 0.9203 hypothetical 0.9079 0.8654 0.8862 possible 0.7 0.6146 0.6545 macro-avg 0.8496 0.7991 0.8236 micro-avg 0.9245 0.9245 0.9245 ``` --- layout: model title: English asr_wav2vec2_base_100h_ngram TFWav2Vec2ForCTC from saahith author: John Snow Labs name: asr_wav2vec2_base_100h_ngram date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_ngram` is a English model originally trained by saahith. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_100h_ngram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_ngram_en_4.2.0_3.0_1664042339482.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_ngram_en_4.2.0_3.0_1664042339482.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_100h_ngram", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_100h_ngram", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_100h_ngram| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: IndicBERT - Albert for 12 major Indian languages author: John Snow Labs name: albert_indic date: 2022-01-26 tags: [open_source, albert, as, bn, en, gu, kn, ml, mr, or, pa, ta, te, xx] task: Embeddings language: xx edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IndicBERT is a multilingual ALBERT model pretrained exclusively on 12 major Indian languages. It is pre-trained on our novel monolingual corpus of around 9 billion tokens and subsequently evaluated on a set of diverse tasks. IndicBERT has much fewer parameters than other multilingual models (mBERT, XLM-R etc.) while it also achieves a performance on-par or better than these models. The 12 languages covered by IndicBERT are: Assamese, Bengali, English, Gujarati, Hindi, Kannada, Malayalam, Marathi, Oriya, Punjabi, Tamil, Telugu. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_indic_xx_3.4.0_3.0_1643211494926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_indic_xx_3.4.0_3.0_1643211494926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from sparknlp.base import * from sparknlp.annotator import * from pyspark.ml import Pipeline documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") embeddings = AlbertEmbeddings.pretrained("albert_indic","xx") \ .setInputCols(["document",'token'])\ .setOutputCol("embeddings")\ embeddingsFinisher = EmbeddingsFinisher() \ .setInputCols(["embeddings"]) \ .setOutputCols("finished_embeddings") \ .setOutputAsVector(True) \ .setCleanAnnotations(False) pipeline = Pipeline().setStages([ documentAssembler, tokenizer, embeddings, embeddingsFinisher ]) data = spark.createDataFrame([ ["கர்நாடக சட்டப் பேரவையில் வெற்றி பெற்ற எம்எல்ஏக்கள் இன்று பதவியேற்றுக் கொண்ட நிலையில் , காங்கிரஸ் எம்எல்ஏ ஆனந்த் சிங் க்கள் ஆப்சென்ட் ஆகி அதிர்ச்சியை ஏற்படுத்தியுள்ளார் . உச்சநீதிமன்ற உத்தரவுப்படி இன்று மாலை முதலமைச்சர் எடியூரப்பா இன்று நம்பிக்கை வாக்கெடுப்பு நடத்தி பெரும்பான்மையை நிரூபிக்க உச்சநீதிமன்றம் உத்தரவிட்டது ."], ]).toDF("text") result = pipeline.fit(data).transform(data) result.selectExpr("explode(finished_embeddings) as result").show(5, 80) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.Tokenizer import com.johnsnowlabs.nlp.embeddings.AlbertEmbeddings import com.johnsnowlabs.nlp.EmbeddingsFinisher import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = AlbertEmbeddings.pretrained("albert_indic", "xx") .setInputCols("token", "document") .setOutputCol("embeddings") val embeddingsFinisher = new EmbeddingsFinisher() .setInputCols("embeddings") .setOutputCols("finished_embeddings") .setOutputAsVector(true) .setCleanAnnotations(false) val pipeline = new Pipeline().setStages(Array( documentAssembler, tokenizer, embeddings, embeddingsFinisher )) val data = Seq("கர்நாடக சட்டப் பேரவையில் வெற்றி பெற்ற எம்எல்ஏக்கள் இன்று பதவியேற்றுக் கொண்ட நிலையில் , காங்கிரஸ் எம்எல்ஏ ஆனந்த் சிங் க்கள் ஆப்சென்ட் ஆகி அதிர்ச்சியை ஏற்படுத்தியுள்ளார் . உச்சநீதிமன்ற உத்தரவுப்படி இன்று மாலை முதலமைச்சர் எடியூரப்பா இன்று நம்பிக்கை வாக்கெடுப்பு நடத்தி பெரும்பான்மையை நிரூபிக்க உச்சநீதிமன்றம் உத்தரவிட்டது .") .toDF("text") val result = pipeline.fit(data).transform(data) result.selectExpr("explode(finished_embeddings) as result").show(5, 80) ``` {:.nlu-block} ```python import nlu nlu.load("xx.embed.albert.indic").predict("""கர்நாடக சட்டப் பேரவையில் வெற்றி பெற்ற எம்எல்ஏக்கள் இன்று பதவியேற்றுக் கொண்ட நிலையில் , காங்கிரஸ் எம்எல்ஏ ஆனந்த் சிங் க்கள் ஆப்சென்ட் ஆகி அதிர்ச்சியை ஏற்படுத்தியுள்ளார் . உச்சநீதிமன்ற உத்தரவுப்படி இன்று மாலை முதலமைச்சர் எடியூரப்பா இன்று நம்பிக்கை வாக்கெடுப்பு நடத்தி பெரும்பான்மையை நிரூபிக்க உச்சநீதிமன்றம் உத்தரவிட்டது .""") ```

## Results ```bash +--------------------------------------------------------------------------------+ | result| +--------------------------------------------------------------------------------+ |[0.2693195641040802,-0.6446362733840942,-0.05138964205980301,0.06030936539173...| |[0.027906809002161026,-0.37459731101989746,-0.08371371030807495,-0.0869174525...| |[0.3804604113101959,-0.7870151400566101,0.08463867008686066,-0.30186718702316...| |[0.15204764902591705,-0.26839596033096313,0.07375998795032501,-0.131638795137...| |[0.1482795625925064,-0.221298485994339,-0.022987276315689087,-0.2132280170917...| +--------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_indic| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|128.3 MB| ## References The model was exported from transformers and is based on https://github.com/AI4Bharat/indic-bert --- layout: model title: English RobertaForQuestionAnswering (from comacrae) author: John Snow Labs name: roberta_qa_roberta_paraphrasev3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-paraphrasev3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_paraphrasev3_en_4.0.0_3.0_1655738199528.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_paraphrasev3_en_4.0.0_3.0_1655738199528.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_paraphrasev3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_paraphrasev3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.paraphrasev3.by_comacrae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_paraphrasev3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/comacrae/roberta-paraphrasev3 --- layout: model title: English image_classifier_vit_base_patch32_384_finetuned_eurosat ViTForImageClassification from keithanpai author: John Snow Labs name: image_classifier_vit_base_patch32_384_finetuned_eurosat date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch32_384_finetuned_eurosat` is a English model originally trained by keithanpai. ## Predicted Entities `dff`, `bklf`, `nvf`, `vascf`, `akiecf`, `bccf`, `melf` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch32_384_finetuned_eurosat_en_4.1.0_3.0_1660172185841.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch32_384_finetuned_eurosat_en_4.1.0_3.0_1660172185841.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch32_384_finetuned_eurosat", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch32_384_finetuned_eurosat", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch32_384_finetuned_eurosat| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|328.4 MB| --- layout: model title: Part of Speech for Irish author: John Snow Labs name: pos_ud_idt date: 2021-03-09 tags: [part_of_speech, open_source, irish, pos_ud_idt, ga] task: Part of Speech Tagging language: ga edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADP - NOUN - DET - AUX - PRON - VERB - SCONJ - PART - ADV - PUNCT - CCONJ - ADJ - PROPN - NUM - X - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_idt_ga_3.0.0_3.0_1615292201208.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_idt_ga_3.0.0_3.0_1615292201208.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_idt", "ga") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Dia duit ó John Labs Sneachta! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_idt", "ga") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Dia duit ó John Labs Sneachta! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Dia duit ó John Labs Sneachta! ""] token_df = nlu.load('ga.pos').predict(text) token_df ```

## Results ```bash token pos 0 Dia NOUN 1 duit NOUN 2 ó ADP 3 John PROPN 4 Labs PROPN 5 Sneachta NOUN 6 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_idt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|ga| --- layout: model title: BioBERT Embeddings (Pubmed Large) author: John Snow Labs name: biobert_pubmed_large_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pubmed_large_cased_en_2.6.2_2.4_1600529365263.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pubmed_large_cased_en_2.6.2_2.4_1600529365263.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("biobert_pubmed_large_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("biobert_pubmed_large_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.pubmed_large_cased').predict(text, output_level='token') embeddings_df ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_1", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_1", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_1| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Laws Spanish Named Entity Recognition (from `hackathon-pln-es`) author: John Snow Labs name: roberta_ner_jurisbert_finetuning_ner date: 2022-05-20 tags: [roberta, ner, token_classification, es, open_source] task: Named Entity Recognition language: es edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `jurisbert-finetuning-ner` is a Spanish model orginally trained by `hackathon-pln-es`. ## Predicted Entities `TRAT_INTL`, `LEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_jurisbert_finetuning_ner_es_3.4.4_3.0_1653046369327.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_jurisbert_finetuning_ner_es_3.4.4_3.0_1653046369327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_jurisbert_finetuning_ner| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://huggingface.co/hackathon-pln-es/jurisbert-finetuning-ner --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Rocketknight1) author: John Snow Labs name: distilbert_qa_rocketknight1_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Rocketknight1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_rocketknight1_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769088913.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_rocketknight1_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769088913.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rocketknight1_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rocketknight1_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_rocketknight1_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Rocketknight1/distilbert-base-uncased-finetuned-squad --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from gbennett) author: John Snow Labs name: xlmroberta_ner_gbennett_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `gbennett`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_gbennett_base_finetuned_panx_de_4.1.0_3.0_1660433217069.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_gbennett_base_finetuned_panx_de_4.1.0_3.0_1660433217069.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_gbennett_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_gbennett_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_gbennett_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/gbennett/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Translate English to West Germanic languages Pipeline author: John Snow Labs name: translate_en_gmw date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, gmw, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `gmw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_gmw_xx_2.7.0_2.4_1609689808163.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_gmw_xx_2.7.0_2.4_1609689808163.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_gmw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_gmw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.gmw').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_gmw| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Fees and expenses Clause Binary Classifier (md) author: John Snow Labs name: legclf_fees_and_expenses_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `fees-and-expenses` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `fees-and-expenses` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_fees_and_expenses_md_en_1.0.0_3.0_1673460290897.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_fees_and_expenses_md_en_1.0.0_3.0_1673460290897.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_fees_and_expenses_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[fees-and-expenses]| |[other]| |[other]| |[fees-and-expenses]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_fees_and_expenses_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support miscellaneous-provisions 0.75 0.75 0.75 24 other 0.85 0.85 0.85 39 accuracy 0.81 63 macro avg 0.80 0.80 0.80 63 weighted avg 0.81 0.81 0.81 63 ``` --- layout: model title: Pipeline to Detect Drugs - Generalized Single Entity (ner_drugs_greedy) author: John Snow Labs name: ner_drugs_greedy_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_drugs_greedy](https://nlp.johnsnowlabs.com/2021/03/31/ner_drugs_greedy_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugs_greedy_pipeline_en_4.3.0_3.2_1678877919575.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugs_greedy_pipeline_en_4.3.0_3.2_1678877919575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_drugs_greedy_pipeline", "en", "clinical/models") text = '''DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_drugs_greedy_pipeline", "en", "clinical/models") val text = "DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.drugs_greedy.pipeline").predict("""DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------------------|--------:|------:|:------------|-------------:| | 0 | hydrocortisone tablets | 48 | 69 | DRUG | 0.9923 | | 1 | 20 mg to 240 mg of hydrocortisone | 85 | 117 | DRUG | 0.7361 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_greedy_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal Titles And Subtitles Clause Binary Classifier author: John Snow Labs name: legclf_titles_and_subtitles_clause date: 2023-01-29 tags: [en, legal, classification, titles, subtitles, clauses, titles_and_subtitles, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `titles-and-subtitles` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `titles-and-subtitles`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_titles_and_subtitles_clause_en_1.0.0_3.0_1674993674002.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_titles_and_subtitles_clause_en_1.0.0_3.0_1674993674002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_titles_and_subtitles_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[titles-and-subtitles]| |[other]| |[other]| |[titles-and-subtitles]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_titles_and_subtitles_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label recision recall f1-score support other 0.97 1.00 0.99 39 titles-and-subtitles 1.00 0.97 0.98 30 accuracy - - 0.99 69 macro-avg 0.99 0.98 0.99 69 weighted-avg 0.99 0.99 0.99 69 ``` --- layout: model title: Sentence Entity Resolver for ICD10-CM (Augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_augmented date: 2022-01-21 tags: [icd10cm, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it has been augmented with synonyms for making it more accurate. ## Predicted Entities `ICD10CM Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_3.0_1642756161477.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_3.0_1642756161477.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk| entity|icd10cm_code| resolutions| all_codes| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O2441|gestational diabetes mellitus:::postpartum gestational diabetes mel...| O2441:::O2443:::Z8632:::Z875:::O2431:::O2411:::O244:::O241:::O2481| |subsequent type two diabetes mellitus|PROBLEM| O2411|pre-existing type 2 diabetes mellitus:::disorder associated with ty...|O2411:::E118:::E11:::E139:::E119:::E113:::E1144:::Z863:::Z8639:::E1...| | T2DM|PROBLEM| E11|type 2 diabetes mellitus:::disorder associated with type 2 diabetes...|E11:::E118:::E119:::O2411:::E109:::E139:::E113:::E8881:::Z833:::D64...| | HTG-induced pancreatitis|PROBLEM| K8520|alcohol-induced pancreatitis:::drug-induced acute pancreatitis:::he...|K8520:::K853:::K8590:::F102:::K852:::K859:::K8580:::K8591:::K858:::...| | acute hepatitis|PROBLEM| K720|acute hepatitis:::acute hepatitis a:::acute infectious hepatitis:::...|K720:::B15:::B179:::B172:::Z0389:::B159:::B150:::B16:::K752:::K712:...| | obesity|PROBLEM| E669|obesity:::abdominal obesity:::obese:::central obesity:::overweight ...|E669:::E668:::Z6841:::Q130:::E66:::E6601:::Z8639:::E349:::H3550:::Z...| | a body mass index|PROBLEM| Z6841|finding of body mass index:::observation of body mass index:::mass ...|Z6841:::E669:::R229:::Z681:::R223:::R221:::Z68:::R222:::R220:::R418...| | polyuria|PROBLEM| R35|polyuria:::nocturnal polyuria:::polyuric state:::polyuric state (di...|R35:::R3581:::R358:::E232:::R31:::R350:::R8299:::N401:::E723:::O048...| | polydipsia|PROBLEM| R631|polydipsia:::psychogenic polydipsia:::primary polydipsia:::psychoge...|R631:::F6389:::E232:::F639:::O40:::G475:::M7989:::R632:::R061:::H53...| | poor appetite|PROBLEM| R630|poor appetite:::poor feeding:::bad taste in mouth:::unpleasant tast...|R630:::P929:::R438:::R432:::E86:::R196:::F520:::Z724:::R0689:::Z768...| | vomiting|PROBLEM| R111|vomiting:::intermittent vomiting:::vomiting symptoms:::periodic vom...| R111:::R11:::R1110:::G43A1:::P921:::P9209:::G43A:::R1113:::R110| | a respiratory tract infection|PROBLEM| J988|respiratory tract infection:::upper respiratory tract infection:::b...|J988:::J069:::A499:::J22:::J209:::Z593:::T17:::J0410:::Z1383:::J189...| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|1.4 GB| |Case sensitive:|false| ## Data Source Trained on ICD10CM 2022 Codes dataset: https://www.cdc.gov/nchs/icd/icd10cm.htm --- layout: model title: Relation extraction between Drugs and ADE author: John Snow Labs name: re_ade_clinical date: 2021-07-12 tags: [licensed, clinical, en, relation_extraction, ade] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.1.2 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is capable of Relating Drugs and adverse reactions caused by them; It predicts if an adverse event is caused by a drug or not. `1` : Shows the adverse event and drug entities are related, `0` : Shows the adverse event and drug entities are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/RE_ADE.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_ade_clinical_en_3.1.2_3.0_1626104637779.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_ade_clinical_en_3.1.2_3.0_1626104637779.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `re_ade_clinical` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:---------------:|:--------------:|:----------------:|------------------------------| | re_ade_clinical | 0
1 | ner_ade_clinical | ["ade-drug",
"drug-ade"] |

## Results ```bash | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---------:|:--------|--------------:|------------:|:----------|:--------|--------------:|------------:|:---------------|-----------:| | 1 | DRUG | 12 | 18 | Lipitor | ADE | 52 | 65 | severe fatigue | 1 | | 1 | DRUG | 97 | 105 | voltarene | ADE | 144 | 156 | muscle cramps | 0.997283 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_ade_clinical| |Type:|re| |Compatibility:|Healthcare NLP 3.1.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| ## Data Source This model is trained on custom data annotated by JSL. ## Benchmarking ```bash label precision recall f1-score support 0 0.86 0.88 0.87 1787 1 0.92 0.90 0.91 2586 micro-avg 0.89 0.89 0.89 4373 macro-avg 0.89 0.89 0.89 4373 weighted-avg 0.89 0.89 0.89 4373 ``` --- layout: model title: English asr_model_4 TFWav2Vec2ForCTC from niclas author: John Snow Labs name: pipeline_asr_model_4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_4` is a English model originally trained by niclas. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_model_4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_4_en_4.2.0_3.0_1664098319002.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_4_en_4.2.0_3.0_1664098319002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_model_4', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_model_4", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_model_4| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_Wav2Vec2_XLSR_Bengali_10500 TFWav2Vec2ForCTC from shoubhik author: John Snow Labs name: pipeline_asr_Wav2Vec2_XLSR_Bengali_10500 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_XLSR_Bengali_10500` is a English model originally trained by shoubhik. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Wav2Vec2_XLSR_Bengali_10500_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Wav2Vec2_XLSR_Bengali_10500_en_4.2.0_3.0_1664105201102.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Wav2Vec2_XLSR_Bengali_10500_en_4.2.0_3.0_1664105201102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Wav2Vec2_XLSR_Bengali_10500', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Wav2Vec2_XLSR_Bengali_10500", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Wav2Vec2_XLSR_Bengali_10500| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|3.6 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Brazilian Portuguese NER for Laws (Base) author: John Snow Labs name: legner_br_base date: 2022-09-27 tags: [pt, licensed] task: Named Entity Recognition language: pt edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Deep Learning Portuguese Named Entity Recognition model for the legal domain, trained using Base Bert Embeddings, and is able to predict the following entities: - ORGANIZACAO (Organizations) - JURISPRUDENCIA (Jurisprudence) - PESSOA (Person) - TEMPO (Time) - LOCAL (Location) - LEGISLACAO (Laws) - O (Other) You can find different versions of this model in Models Hub: - With a Deep Learning architecture (non-transformer) and Base Embeddings; - With a Deep Learning architecture (non-transformer) and Large Embeddings; - With a Transformers Architecture and Base Embeddings; - With a Transformers Architecture and Large Embeddings; ## Predicted Entities `PESSOA`, `ORGANIZACAO`, `LEGISLACAO`, `JURISPRUDENCIA`, `TEMPO`, `LOCAL` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_LEGAL_PT/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_br_base_pt_1.0.0_3.0_1664276774137.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_br_base_pt_1.0.0_3.0_1664276774137.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+----------------+ | token| ner| +-------------------+----------------+ | diante| O| | do| O| | exposto| O| | ,| O| | com| O| | fundamento| O| | nos| O| | artigos| B-LEGISLACAO| | 32| I-LEGISLACAO| | ,| I-LEGISLACAO| | i| I-LEGISLACAO| | ,| I-LEGISLACAO| | e| I-LEGISLACAO| | 33| I-LEGISLACAO| | ,| I-LEGISLACAO| | da| I-LEGISLACAO| | lei| I-LEGISLACAO| | 8.443/1992| I-LEGISLACAO| | ,| O| | submetem-se| O| | os| O| | autos| O| | à| O| | consideração| O| | superior| O| | ,| O| | com| O| | posterior| O| | encaminhamento| O| | ao| O| | ministério| B-ORGANIZACAO| | público| I-ORGANIZACAO| | junto| O| | ao| O| | tcu| B-ORGANIZACAO| | e| O| | ao| O| | gabinete| O| | do| O| | relator| O| | ,| O| | propondo| O| | :| O| | a| O| | )| O| | conhecer| O| | do| O| | recurso| O| | e| O| | ,| O| | no| O| | mérito| O| | ,| O| | negar-lhe| O| | provimento| O| | ;| O| | b| O| | )| O| | comunicar| O| | ao| O| | recorrente| O| | ,| O| | ao| O| | superior| B-ORGANIZACAO| | tribunal| I-ORGANIZACAO| | militar| I-ORGANIZACAO| | e| O| | ao| O| | tribunal| B-ORGANIZACAO| | regional| I-ORGANIZACAO| | federal| I-ORGANIZACAO| | da| I-ORGANIZACAO| | 2ª| I-ORGANIZACAO| | região| I-ORGANIZACAO| | ,| O| | a| O| | fim| O| | de| O| | fornecer| O| | subsídios| O| | para| O| | os| O| | processos| O| | judiciais| O| |2001.34.00.024796-9|B-JURISPRUDENCIA| | e| O| |2003.34.00.044227-3|B-JURISPRUDENCIA| | ;| O| | e| O| | aos| O| | demais| O| | interessados| O| | a| O| | deliberação| O| | que| O| | vier| O| | a| O| | ser| O| | proferida| O| | por| O| | esta| O| | corte| O| | ”| O| | .| O| +-------------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_br_base| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Size:|18.8 MB| ## References Original texts available in https://paperswithcode.com/sota?task=Token+Classification&dataset=lener_br and in-house data augmentation with weak labelling ## Benchmarking ```bash label precision recall f1-score support B-JURISPRUDENCIA 0.84 0.91 0.88 175 B-LEGISLACAO 0.96 0.96 0.96 347 B-LOCAL 0.69 0.68 0.68 40 B-ORGANIZACAO 0.95 0.71 0.81 441 B-PESSOA 0.91 0.95 0.93 221 B-TEMPO 0.94 0.86 0.90 176 I-JURISPRUDENCIA 0.86 0.91 0.89 461 I-LEGISLACAO 0.98 0.99 0.98 2012 I-LOCAL 0.54 0.53 0.53 72 I-ORGANIZACAO 0.94 0.76 0.84 768 I-PESSOA 0.93 0.98 0.95 461 I-TEMPO 0.90 0.85 0.88 66 O 0.99 1.00 0.99 38419 accuracy - - 0.98 43659 macro-avg 0.88 0.85 0.86 43659 weighted-avg 0.98 0.98 0.98 43659 ``` --- layout: model title: English asr_wav2vec2_base_100h_by_vuiseng9 TFWav2Vec2ForCTC from vuiseng9 author: John Snow Labs name: asr_wav2vec2_base_100h_by_vuiseng9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_by_vuiseng9` is a English model originally trained by vuiseng9. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_100h_by_vuiseng9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_by_vuiseng9_en_4.2.0_3.0_1664022833525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_by_vuiseng9_en_4.2.0_3.0_1664022833525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_100h_by_vuiseng9", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_100h_by_vuiseng9", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_100h_by_vuiseng9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: Arabic BertForMaskedLM Base Cased model (from CAMeL-Lab) author: John Snow Labs name: bert_embeddings_base_arabic_camel_msa_eighth date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabic-camelbert-msa-eighth` is a Arabic model originally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_eighth_ar_4.2.4_3.0_1670016070157.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_eighth_ar_4.2.4_3.0_1670016070157.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa_eighth","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa_eighth","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabic_camel_msa_eighth| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-eighth - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: Detect Cancer Genetics (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_bionlp date: 2021-11-03 tags: [bertfortokenclassification, ner, bionlp, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts biological and genetics terms in cancer-related texts using pre-trained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `Amino_acid`, `Anatomical_system`, `Cancer`, `Cell`, `Cellular_component`, `Developing_anatomical_Structure`, `Gene_or_gene_product`, `Immaterial_anatomical_entity`, `Multi-tissue_structure`, `Organ`, `Organism`, `Organism_subdivision`, `Simple_chemical`, `Tissue`, `Organism_substance`, `Pathological_formation` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bionlp_en_3.3.0_2.4_1635952712612.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bionlp_en_3.3.0_2.4_1635952712612.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+----------------------+ |chunk |ner_label | +-------------------+----------------------+ |erbA IRES |Organism | |erbA/myb virus |Organism | |erythroid cells |Cell | |bone marrow |Multi-tissue_structure| |blastoderm cultures|Cell | |erbA/myb IRES virus|Organism | |erbA IRES virus |Organism | |blastoderm |Cell | +-------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bionlp| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|256| ## Data Source Trained on Cancer Genetics (CG) task of the BioNLP Shared Task 2013. https://aclanthology.org/W13-2008/ ## Benchmarking ```bash label precision recall f1-score support B-Cancer 0.88 0.82 0.85 924 B-Cell 0.84 0.86 0.85 1013 B-Cellular_component 0.87 0.84 0.86 180 B-Developing_anatomical_structure 0.65 0.65 0.65 17 B-Gene_or_gene_product 0.62 0.79 0.69 2520 B-Immaterial_anatomical_entity 0.68 0.74 0.71 31 B-Multi-tissue_structure 0.84 0.76 0.80 303 B-Organ 0.78 0.74 0.76 156 B-Organism 0.93 0.86 0.89 518 B-Organism_subdivision 0.74 0.51 0.61 39 B-Organism_substance 0.93 0.66 0.77 102 B-Pathological_formation 0.85 0.60 0.71 88 B-Simple_chemical 0.61 0.75 0.68 727 B-Tissue 0.74 0.83 0.78 184 I-Amino_acid 0.60 1.00 0.75 3 I-Cancer 0.91 0.69 0.78 604 I-Cell 0.98 0.74 0.84 1091 I-Cellular_component 0.88 0.62 0.73 69 I-Multi-tissue_structure 0.89 0.86 0.87 162 I-Organ 0.67 0.59 0.62 17 I-Organism 0.84 0.45 0.59 120 I-Organism_substance 0.80 0.50 0.62 24 I-Pathological_formation 0.81 0.56 0.67 39 I-Tissue 0.83 0.86 0.84 111 accuracy - - 0.64 12129 macro-avg 0.73 0.56 0.60 12129 weighted-avg 0.83 0.64 0.68 12129 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from en) author: John Snow Labs name: distilbert_qa_squad_base_uncased_finetuned date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `en`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_base_uncased_finetuned_en_4.3.0_3.0_1672770716210.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_base_uncased_finetuned_en_4.3.0_3.0_1672770716210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_base_uncased_finetuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_base_uncased_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_base_uncased_finetuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/en/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering model (from datarpit) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_natural_questions date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-natural-questions` is a English model originally trained by `datarpit`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_natural_questions_en_4.0.0_3.0_1654723994546.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_natural_questions_en_4.0.0_3.0_1654723994546.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_natural_questions","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_natural_questions","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_uncased.by_datarpit").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_natural_questions| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/datarpit/distilbert-base-uncased-finetuned-natural-questions --- layout: model title: French CamemBert Embeddings (from Sebu) author: John Snow Labs name: camembert_embeddings_Sebu_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Sebu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Sebu_generic_model_fr_3.4.4_3.0_1653986850148.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Sebu_generic_model_fr_3.4.4_3.0_1653986850148.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Sebu_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Sebu/dummy-model --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Thitaree) author: John Snow Labs name: distilbert_qa_thitaree_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Thitaree`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_thitaree_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769425321.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_thitaree_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769425321.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_thitaree_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_thitaree_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_thitaree_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Thitaree/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-27 tags: [pt, licensed, ner, legal, mapa] task: Named Entity Recognition language: pt edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Portuguese` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_pt_1.0.0_3.0_1682608680085.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_pt_1.0.0_3.0_1682608680085.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_base_pt_cased", "pt")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "pt", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Nos termos dos Decretos da Garda Síochána (6), só pode ser admitido como estagiário para integrar a força policial nacional quem tiver pelo menos 18 anos, mas menos de 35 anos de idade, no primeiro dia do mês em que tenha sido publicado pela primeira vez, num jornal nacional, o anúncio da vaga a que o recrutamento respeita."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-----------------------+------------+ |chunk |ner_label | +-----------------------+------------+ |Garda Síochána |ORGANISATION| |força policial nacional|ORGANISATION| |18 anos |AMOUNT | |35 anos |AMOUNT | +-----------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.91 0.91 0.91 23 AMOUNT 1.00 0.83 0.91 6 DATE 1.00 0.95 0.97 61 ORGANISATION 0.85 0.77 0.81 30 PERSON 0.88 0.91 0.89 65 macro-avg 0.92 0.90 0.91 185 macro-avg 0.93 0.87 0.90 185 weighted-avg 0.92 0.90 0.91 185 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from nlpunibo) Config1 author: John Snow Labs name: distilbert_qa_base_config1 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert_base_config1` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config1_en_4.0.0_3.0_1654727786120.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config1_en_4.0.0_3.0_1654727786120.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_config1.by_nlpunibo").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_config1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/distilbert_base_config1 --- layout: model title: Translate English to Western Malayo-Polynesian languages Pipeline author: John Snow Labs name: translate_en_pqw date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, pqw, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `pqw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_pqw_xx_2.7.0_2.4_1609688594063.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_pqw_xx_2.7.0_2.4_1609688594063.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_pqw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_pqw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.pqw').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_pqw| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Chunk Entity Resolver RxNorm-scdc author: John Snow Labs name: chunkresolve_rxnorm_in_healthcare date: 2021-04-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to RxNorm codes using chunk embeddings (augmented with synonyms, four times richer than previous resolver). ## Predicted Entities RxNorm codes {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_RXNORM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_in_healthcare_en_3.0.0_3.0_1618605195699.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_in_healthcare_en_3.0.0_3.0_1618605195699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... resolver = ChunkEntityResolverModel.pretrained("chunkresolve_rxnorm_in_healthcare","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") pipeline = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, resolver]) data = spark.createDataFrame([["A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation."]]).toDF("text") model = pipeline.fit(data) results = model.transform(data) ... ``` ```scala ... val resolver = ChunkEntityResolverModel.pretrained("chunkresolve_rxnorm_in_healthcare","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(document_assembler, sbert_embedder, resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation.").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | chunk| entity| target_text| code|confidence| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | metformin|TREATMENT|metFORMIN compounding powder:::Metformin Hydrochloride Powder:::metFORMIN 500 mg oral tablet:::me...| 601021| 0.2364| | glipizide|TREATMENT|Glipizide Powder:::Glipizide Crystal:::Glipizide Tablets:::glipiZIDE 5 mg oral tablet:::glipiZIDE...| 241604| 0.3647| |dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG|TREATMENT|Ezetimibe and Atorvastatin Tablets:::Amlodipine and Atorvastatin Tablets:::Atorvastatin Calcium T...|1422084| 0.3407| | dapagliflozin|TREATMENT|Dapagliflozin Tablets:::dapagliflozin 5 mg oral tablet:::dapagliflozin 10 mg oral tablet:::Dapagl...|1488568| 0.7070| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_rxnorm_in_healthcare| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| --- layout: model title: Finnish asr_wav2vec2_xlsr_300m_finnish TFWav2Vec2ForCTC from aapot author: John Snow Labs name: asr_wav2vec2_xlsr_300m_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_300m_finnish` is a Finnish model originally trained by aapot. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_300m_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_300m_finnish_fi_4.2.0_3.0_1664023005420.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_300m_finnish_fi_4.2.0_3.0_1664023005420.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_300m_finnish", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_300m_finnish", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_300m_finnish| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: English DistilBertForQuestionAnswering model (from graviraja) author: John Snow Labs name: distilbert_qa_graviraja_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `graviraja`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_graviraja_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725306055.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_graviraja_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725306055.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_graviraja_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_graviraja_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_graviraja").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_graviraja_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/graviraja/distilbert-base-uncased-finetuned-squad --- layout: model title: Indonesian RoBERTa Embeddings (from w11wo) author: John Snow Labs name: roberta_embeddings_indo_roberta_small date: 2022-04-14 tags: [roberta, embeddings, id, open_source] task: Embeddings language: id edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indo-roberta-small` is a Indonesian model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indo_roberta_small_id_3.4.2_3.0_1649948731693.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indo_roberta_small_id_3.4.2_3.0_1649948731693.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indo_roberta_small| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|id| |Size:|314.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/indo-roberta-small - https://arxiv.org/abs/1907.11692 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_orkg_which_1e_04 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-orkg-which-1e-04` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_which_1e_04_en_4.3.0_3.0_1672766889512.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_which_1e_04_en_4.3.0_3.0_1672766889512.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_which_1e_04","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_which_1e_04","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_orkg_which_1e_04| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Moussab/distilbert-base-cased-distilled-squad-orkg-which-1e-04 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_anan0329 TFWav2Vec2ForCTC from anan0329 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_anan0329` is a English model originally trained by anan0329. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329_en_4.2.0_3.0_1664114693280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329_en_4.2.0_3.0_1664114693280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_anan0329| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese Bert Embeddings (Base) author: John Snow Labs name: bert_embeddings_jdt_fin_roberta_wwm date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `jdt-fin-roberta-wwm` is a Chinese model orginally trained by `wangfan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_jdt_fin_roberta_wwm_zh_3.4.2_3.0_1649669984329.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_jdt_fin_roberta_wwm_zh_3.4.2_3.0_1649669984329.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_jdt_fin_roberta_wwm| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/wangfan/jdt-fin-roberta-wwm - https://3.cn/103c-hwSS - https://3.cn/103c-izpe --- layout: model title: English image_classifier_vit_llama_alpaca_guanaco_vicuna ViTForImageClassification from osanseviero author: John Snow Labs name: image_classifier_vit_llama_alpaca_guanaco_vicuna date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_llama_alpaca_guanaco_vicuna` is a English model originally trained by osanseviero. ## Predicted Entities `alpaca`, `guanaco`, `llama`, `vicuna` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_llama_alpaca_guanaco_vicuna_en_4.1.0_3.0_1660166270042.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_llama_alpaca_guanaco_vicuna_en_4.1.0_3.0_1660166270042.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_llama_alpaca_guanaco_vicuna", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_llama_alpaca_guanaco_vicuna", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_llama_alpaca_guanaco_vicuna| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Sublease Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_sublease_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, sublease, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_sublease_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `sublease-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `sublease-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sublease_agreement_en_1.0.0_3.0_1668117647287.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sublease_agreement_en_1.0.0_3.0_1668117647287.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[sublease-agreement]| |[other]| |[other]| |[sublease-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sublease_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 1.00 0.99 66 sublease-agreement 1.00 0.97 0.99 35 accuracy - - 0.99 101 macro-avg 0.99 0.99 0.99 101 weighted-avg 0.99 0.99 0.99 101 ``` --- layout: model title: German Financial Bert Word Embeddings author: John Snow Labs name: bert_embeddings_german_financial_statements_bert date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Financial Bert Word Embeddings model, trained on German Financial Statements. Uploaded to Hugging Face, adapted and imported into Spark NLP. `german-financial-statements-bert` is a German model orginally trained by `fabianrausch`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_german_financial_statements_bert_de_3.4.2_3.0_1649676227862.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_german_financial_statements_bert_de_3.4.2_3.0_1649676227862.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_german_financial_statements_bert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|409.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/fabianrausch/german-financial-statements-bert --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes (Clinical Drug) author: John Snow Labs name: sbiobertresolve_umls_clinical_drugs date: 2022-07-05 tags: [entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities to UMLS CUI codes. It is trained on 2022AA UMLS dataset. The complete dataset has 127 different categories, and this model is trained on the Clinical Drug category using sbiobert_base_cased_mli embeddings. ## Predicted Entities `Predicts UMLS codes for Clinical Drug medical concepts` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_clinical_drugs_en_4.0.0_3.0_1657039242193.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_clinical_drugs_en_4.0.0_3.0_1657039242193.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("clinical_ner") ner_model_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "clinical_ner"])\ .setOutputCol("ner_chunk") chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_umls_clinical_drugs","en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline = Pipeline(stages = [document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunk2doc, sbert_embedder, resolver]) data = spark.createDataFrame([["""She was immediately given hydrogen peroxide 30 mg to treat the infection on her leg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg."""]]).toDF("text") results = pipeline.fit(data).transform(data) ``` ```scala ... val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel .pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("clinical_ner") val ner_model_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "clinical_ner")) .setOutputCol("ner_chunk") val chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") .setCaseSensitive(False) val resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_umls_clinical_drugs", "en", "clinical/models") .setInputCols(Array("ner_chunk_doc", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunk2doc, sbert_embedder, resolver)) val data = Seq("She was immediately given hydrogen peroxide 30 mg to treat the infection on her leg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg.").toDF("text") val res = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.umls_clinical_drugs").predict("""She was immediately given hydrogen peroxide 30 mg to treat the infection on her leg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg.""") ```

## Results ```bash | | chunk | code | code_description | all_k_code_desc | all_k_codes | |---:|:------------------------------|:---------|:---------------------------|:-------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | hydrogen peroxide 30 mg | C1126248 | hydrogen peroxide 30 mg/ml | ['C1126248', 'C0304655', 'C1605252', 'C0304656', 'C1154260'] | ['hydrogen peroxide 30 mg/ml', 'hydrogen peroxide solution 30%', 'hydrogen peroxide 30 mg/ml [proxacol]', 'hydrogen peroxide 30 mg/ml cutaneous solution', 'benzoyl peroxide 30 mg/ml'] | | 1 | Neosporin Cream | C0132149 | neosporin cream | ['C0132149', 'C0358174', 'C0357999', 'C0307085', 'C0698810'] | ['neosporin cream', 'nystan cream', 'nystadermal cream', 'nupercainal cream', 'nystaform cream'] | | 2 | magnesium hydroxide 100mg/1ml | C1134402 | magnesium hydroxide 100 mg | ['C1134402', 'C1126785', 'C4317023', 'C4051486', 'C4047137'] | ['magnesium hydroxide 100 mg', 'magnesium hydroxide 100 mg/ml', 'magnesium sulphate 100mg/ml injection', 'magnesium sulfate 100 mg', 'magnesium sulfate 100 mg/ml'] | | 3 | metformin 1000 mg | C0987664 | metformin 1000 mg | ['C0987664', 'C2719784', 'C0978482', 'C2719786', 'C4282269'] | ['metformin 1000 mg', 'metformin hydrochloride 1000 mg', 'metformin hcl 1000mg tab', 'metformin hydrochloride 1000 mg [fortamet]', 'metformin hcl 1000mg sa tab'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_clinical_drugs| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[name]| |Language:|en| |Size:|2.5 GB| |Case sensitive:|false| ## References Trained on 2022AA UMLS dataset’s Clinical Drug category. https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: Legal Other Definitional Provisions Clause Binary Classifier author: John Snow Labs name: legclf_other_definitional_provisions_clause date: 2023-01-29 tags: [en, legal, classification, other, definitional, provisions, clauses, other_definitional_provisions, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `other-definitional-provisions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other-definitional-provisions`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_other_definitional_provisions_clause_en_1.0.0_3.0_1674993355901.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_other_definitional_provisions_clause_en_1.0.0_3.0_1674993355901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_other_definitional_provisions_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[other-definitional-provisions]| |[other]| |[other]| |[other-definitional-provisions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_other_definitional_provisions_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 1.00 0.98 30 other-definitional-provisions 1.00 0.95 0.98 22 accuracy - - 0.98 52 macro-avg 0.98 0.98 0.98 52 weighted-avg 0.98 0.98 0.98 52 ``` --- layout: model title: Pipeline for Detect Medication author: John Snow Labs name: ner_medication_pipeline date: 2023-06-13 tags: [ner, en, licensed] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained pipeline to detect medication entities. It was built on the top of `ner_posology_greedy` model and also augmented with the drug names mentioned in UK and US drugbank datasets. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_medication_pipeline_en_4.4.4_3.2_1686665836067.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_medication_pipeline_en_4.4.4_3.2_1686665836067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param}

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_medication_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - TextMatcherModel - ChunkMergeModel - Finisher --- layout: model title: Translate English to Finnish Pipeline author: John Snow Labs name: translate_en_fi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, fi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `fi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_fi_xx_2.7.0_2.4_1609689441892.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_fi_xx_2.7.0_2.4_1609689441892.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_fi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_fi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.fi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_fi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Stop Words Cleaner for Greek author: John Snow Labs name: stopwords_el date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: el edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, el] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_el_el_2.5.4_2.4_1594742437880.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_el_el_2.5.4_2.4_1594742437880.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_el", "el") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Εκτός από το ότι είναι ο βασιλιάς του Βορρά, ο John Snow είναι Άγγλος γιατρός και ηγέτης στην ανάπτυξη της αναισθησίας και της ιατρικής υγιεινής.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_el", "el") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Εκτός από το ότι είναι ο βασιλιάς του Βορρά, ο John Snow είναι Άγγλος γιατρός και ηγέτης στην ανάπτυξη της αναισθησίας και της ιατρικής υγιεινής.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Εκτός από το ότι είναι ο βασιλιάς του Βορρά, ο John Snow είναι Άγγλος γιατρός και ηγέτης στην ανάπτυξη της αναισθησίας και της ιατρικής υγιεινής."""] stopword_df = nlu.load('el.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Εκτός', metadata={'sentence': '0'}), Row(annotatorType='token', begin=6, end=8, result='από', metadata={'sentence': '0'}), Row(annotatorType='token', begin=13, end=15, result='ότι', metadata={'sentence': '0'}), Row(annotatorType='token', begin=17, end=21, result='είναι', metadata={'sentence': '0'}), Row(annotatorType='token', begin=25, end=32, result='βασιλιάς', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_el| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|el| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Financial Finbert Sentiment Analysis (DistilRoBerta) author: John Snow Labs name: finclf_distilroberta_sentiment_analysis date: 2022-08-09 tags: [en, finance, sentiment, classification, sentiment_analysis, licensed] task: Sentiment Analysis language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a pre-trained NLP model to analyze sentiment of financial text. It is built by further training the DistilRoBerta language model in the finance domain, using a financial corpus and thereby fine-tuning it for financial sentiment classification. Financial PhraseBank by Malo et al. (2014) and in-house JSL documents and annotations have been used for fine-tuning. ## Predicted Entities `positive`, `negative`, `neutral` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_FINANCE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_distilroberta_sentiment_analysis_en_1.0.0_3.2_1660055192412.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_distilroberta_sentiment_analysis_en_1.0.0_3.2_1660055192412.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = nlp.RoBertaForSequenceClassification.pretrained("finclf_distilroberta_sentiment_analysis","en", "finance/models") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") nlpPipeline = nlp.Pipeline( stages = [ documentAssembler, tokenizer, classifier]) # couple of simple examples example = spark.createDataFrame([["Stocks rallied and the British pound gained."]]).toDF("text") result = nlpPipeline.fit(example).transform(example) # result is a DataFrame result.select("text", "class.result").show() ```

## Results ```bash +--------------------+----------+ | text| result| +--------------------+----------+ |Stocks rallied an...|[positive]| +--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_distilroberta_sentiment_analysis| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|309.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References In-house financial documents and Financial PhraseBank by Malo et al. (2014) ## Benchmarking ```bash label precision recall f1-score support positive 0.77 0.88 0.81 253 negative 0.86 0.85 0.88 133 neutral 0.93 0.86 0.90 584 accuracy - - 0.86 970 macro-avg 0.85 0.86 0.85 970 weighted-avg 0.87 0.86 0.87 970 ``` --- layout: model title: Portuguese BertForTokenClassification Cased model (from pucpr) author: John Snow Labs name: bert_token_classifier_clinicalnerpt_disease date: 2022-11-30 tags: [pt, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: pt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `clinicalnerpt-disease` is a Portuguese model originally trained by `pucpr`. ## Predicted Entities `DiseaseOrSyndrome` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_disease_pt_4.2.4_3.0_1669822418241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_disease_pt_4.2.4_3.0_1669822418241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_disease","pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_disease","pt") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_clinicalnerpt_disease| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|pt| |Size:|665.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/pucpr/clinicalnerpt-disease - https://www.aclweb.org/anthology/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/SemClinBr - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: Stop Words Cleaner for Marathi author: John Snow Labs name: stopwords_mr date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: mr edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, mr] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_mr_mr_2.5.4_2.4_1594742439994.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_mr_mr_2.5.4_2.4_1594742439994.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_mr", "mr") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("उत्तरेचा राजा होण्याव्यतिरिक्त, जॉन स्नो एक इंग्रज चिकित्सक आहे आणि भूल आणि वैद्यकीय स्वच्छतेच्या विकासासाठी अग्रगण्य आहे.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_mr", "mr") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("उत्तरेचा राजा होण्याव्यतिरिक्त, जॉन स्नो एक इंग्रज चिकित्सक आहे आणि भूल आणि वैद्यकीय स्वच्छतेच्या विकासासाठी अग्रगण्य आहे.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""उत्तरेचा राजा होण्याव्यतिरिक्त, जॉन स्नो एक इंग्रज चिकित्सक आहे आणि भूल आणि वैद्यकीय स्वच्छतेच्या विकासासाठी अग्रगण्य आहे."""] stopword_df = nlu.load('mr.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=7, result='उत्तरेचा', metadata={'sentence': '0'}), Row(annotatorType='token', begin=9, end=12, result='राजा', metadata={'sentence': '0'}), Row(annotatorType='token', begin=14, end=29, result='होण्याव्यतिरिक्त', metadata={'sentence': '0'}), Row(annotatorType='token', begin=30, end=30, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=32, end=34, result='जॉन', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_mr| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|mr| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Translate Icelandic to English Pipeline author: John Snow Labs name: translate_is_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, is, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `is` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_is_en_xx_2.7.0_2.4_1609690970413.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_is_en_xx_2.7.0_2.4_1609690970413.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_is_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_is_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.is.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_is_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Xhosa Pipeline author: John Snow Labs name: translate_en_xh date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, xh, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `xh` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_xh_xx_2.7.0_2.4_1609689615747.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_xh_xx_2.7.0_2.4_1609689615747.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_xh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_xh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.xh').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_xh| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from zhenyueyu) author: John Snow Labs name: distilbert_qa_zhenyueyu_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `zhenyueyu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_zhenyueyu_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773369365.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_zhenyueyu_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773369365.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zhenyueyu_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zhenyueyu_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_zhenyueyu_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/zhenyueyu/distilbert-base-uncased-finetuned-squad --- layout: model title: Abkhazian asr_xls_test TFWav2Vec2ForCTC from pere author: John Snow Labs name: pipeline_asr_xls_test date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_test` is a Abkhazian model originally trained by pere. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xls_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_test_ab_4.2.0_3.0_1664020711203.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_test_ab_4.2.0_3.0_1664020711203.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_test', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_test", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|452.5 KB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Arabic Bert Embeddings (Base) author: John Snow Labs name: bert_embeddings_bert_base_qarib date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-qarib` is a Arabic model orginally trained by `qarib`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_qarib_ar_3.4.2_3.0_1649677790858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_qarib_ar_3.4.2_3.0_1649677790858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_qarib| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|506.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/qarib/bert-base-qarib - http://opus.nlpl.eu/ - https://github.com/qcri/QARIB/Training_QARiB.md - https://github.com/qcri/QARIB/Using_QARiB.md --- layout: model title: Catalan, Valencian asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala TFWav2Vec2ForCTC from softcatala author: John Snow Labs name: asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala date: 2022-09-24 tags: [wav2vec2, ca, audio, open_source, asr] task: Automatic Speech Recognition language: ca edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala` is a Catalan, Valencian model originally trained by softcatala. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_ca_4.2.0_3.0_1664037065825.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_ca_4.2.0_3.0_1664037065825.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala", "ca")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala", "ca") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ca| |Size:|1.2 GB| --- layout: model title: Sango asr_wav2vec2_large_xlsr_53_swiss_german TFWav2Vec2ForCTC from Yves author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_swiss_german date: 2022-09-24 tags: [wav2vec2, sg, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: sg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_swiss_german` is a Sango model originally trained by Yves. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_swiss_german_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_swiss_german_sg_4.2.0_3.0_1664022719221.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_swiss_german_sg_4.2.0_3.0_1664022719221.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_swiss_german', lang = 'sg') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_swiss_german", lang = "sg") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_swiss_german| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|sg| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Danish XlmRoBertaForQuestionAnswering (from saattrupdan) author: John Snow Labs name: xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan date: 2022-06-24 tags: [da, open_source, question_answering, xlmroberta] task: Question Answering language: da edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlmr-base-texas-squad-da` is a Danish model originally trained by `saattrupdan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan_da_4.0.0_3.0_1656062061104.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan_da_4.0.0_3.0_1656062061104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan","da") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan","da") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("da.answer_question.squad.xlmr_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlmr_base_texas_squad_da_da_saattrupdan| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|da| |Size:|878.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saattrupdan/xlmr-base-texas-squad-da --- layout: model title: Legal Construction Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_construction_bert date: 2023-03-05 tags: [en, legal, classification, clauses, construction, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Construction` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Construction`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_construction_bert_en_1.0.0_3.0_1678050533068.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_construction_bert_en_1.0.0_3.0_1678050533068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_construction_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Construction]| |[Other]| |[Other]| |[Construction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_construction_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Construction 0.84 0.83 0.84 46 Other 0.88 0.90 0.89 67 accuracy - - 0.87 113 macro-avg 0.86 0.86 0.86 113 weighted-avg 0.87 0.87 0.87 113 ``` --- layout: model title: English BertForTokenClassification Base Uncased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_Original_BiomedNLP_PubMedBERT_base_uncased_abstract date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4_Original-BiomedNLP-PubMedBERT-base-uncased-abstract` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Original_BiomedNLP_PubMedBERT_base_uncased_abstract_en_4.0.0_3.0_1657109212870.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Original_BiomedNLP_PubMedBERT_base_uncased_abstract_en_4.0.0_3.0_1657109212870.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_Original_BiomedNLP_PubMedBERT_base_uncased_abstract| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|false| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4_Original-BiomedNLP-PubMedBERT-base-uncased-abstract --- layout: model title: Multilingual DistilBertForTokenClassification Base Cased model (from mrm8488) author: John Snow Labs name: distilbert_ner_base_multi_cased_finetuned_typo_detection date: 2022-07-21 tags: [open_source, distilbert, ner, typo, multilingual, xx] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT NER model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multi-cased-finetuned-typo-detection` is a Multilingual model originally trained by `mrm8488`. ## Predicted Entities `ok`, `typo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_base_multi_cased_finetuned_typo_detection_xx_4.0.0_3.0_1658399913400.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_base_multi_cased_finetuned_typo_detection_xx_4.0.0_3.0_1658399913400.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_base_multi_cased_finetuned_typo_detection| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://huggingface.co/mrm8488/distilbert-base-multi-cased-finetuned-typo-detection --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6_en_4.0.0_3.0_1655733561065.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6_en_4.0.0_3.0_1655733561065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_64d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|419.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-6 --- layout: model title: Multilingual DistilBertForQuestionAnswering Base Cased model (from monakth) author: John Snow Labs name: distilbert_qa_base_cased_squadv2 date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true recommended: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multilingual-cased-squadv2` is a Multilingual model originally trained by `monakth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_squadv2_xx_4.3.0_3.0_1672767315694.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_squadv2_xx_4.3.0_3.0_1672767315694.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_squadv2","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_squadv2","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_squadv2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/monakth/distilbert-base-multilingual-cased-squadv2 --- layout: model title: Fast Neural Machine Translation Model from Pijin to English author: John Snow Labs name: opus_mt_pis_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pis, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `pis` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_pis_en_xx_2.7.0_2.4_1609163443618.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_pis_en_xx_2.7.0_2.4_1609163443618.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_pis_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Tagalog (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, tl, open_source] task: Embeddings language: tl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tl_3.4.1_3.0_1647461421317.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tl_3.4.1_3.0_1647461421317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_employment_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[employment-agreement]| |[other]| |[other]| |[employment-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_employment_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support employment-agreement 0.96 0.90 0.93 29 other 0.96 0.99 0.98 82 accuracy - - 0.96 111 macro-avg 0.96 0.94 0.95 111 weighted-avg 0.96 0.96 0.96 111 ``` --- layout: model title: Fast Neural Machine Translation Model from Latvian to English author: John Snow Labs name: opus_mt_lv_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, lv, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `lv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_lv_en_xx_2.7.0_2.4_1609163952807.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_lv_en_xx_2.7.0_2.4_1609163952807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_lv_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForSequenceClassification Cased model (from Kaveh8) author: John Snow Labs name: roberta_classifier_autonlp_imdb_rating_625417974 date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification, tensorflow] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-imdb_rating-625417974` is a English model originally trained by `Kaveh8`. ## Predicted Entities `1`, `4`, `3`, `2`, `5` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_autonlp_imdb_rating_625417974_en_4.2.4_3.0_1670622586146.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_autonlp_imdb_rating_625417974_en_4.2.4_3.0_1670622586146.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autonlp_imdb_rating_625417974","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autonlp_imdb_rating_625417974","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_autonlp_imdb_rating_625417974| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|428.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Kaveh8/autonlp-imdb_rating-625417974 --- layout: model title: English asr_wav2vec2_large_xlsr_coraa_portuguese_cv8 TFWav2Vec2ForCTC from lgris author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_coraa_portuguese_cv8` is a English model originally trained by lgris. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_en_4.2.0_3.0_1664043408372.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_en_4.2.0_3.0_1664043408372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_coraa_portuguese_cv8| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chunk Resolver (Cpt Clinical) author: John Snow Labs name: chunkresolve_cpt_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-21 task: Entity Resolution edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities CPT Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_cpt_clinical_en_2.4.5_2.4_1587491373378.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_cpt_clinical_en_2.4.5_2.4_1587491373378.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... cpt_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_cpt_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, cpt_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val cpt_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_cpt_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, cpt_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash chunk entity cpt_description cpt_code 0 a cold, cough PROBLEM Thoracoscopy, surgical; with removal of a sing... 32669 1 runny nose PROBLEM Unlisted procedure, larynx 31599 2 fever PROBLEM Cesarean delivery only; 59514 3 difficulty breathing PROBLEM Repair, laceration of diaphragm, any approach 39501 4 her cough PROBLEM Exploration for postoperative hemorrhage, thro... 35840 5 physical exam TEST Cesarean delivery only; including postpartum care 59515 6 fairly congested PROBLEM Pyelotomy; with drainage, pyelostomy 50125 7 Amoxil TREATMENT Cholecystoenterostomy; with gastroenterostomy 47721 8 Aldex TREATMENT Laparoscopy, surgical; with omentopexy (omenta... 49326 9 difficulty breathing PROBLEM Repair, laceration of diaphragm, any approach 39501 10 more congested PROBLEM for section of 1 or more cranial nerves 61460 11 trouble sleeping PROBLEM Repair, laceration of diaphragm, any approach 39501 12 congestion PROBLEM Repair, laceration of diaphragm, any approach 39501 ``` {:.model-param} ## Model Information {:.table-model} |----------------|---------------------------| | Name: | chunkresolve_cpt_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.2+ | | License: | Licensed | |Edition:|Official| | |Input labels: | token, chunk_embeddings | |Output labels: | entity | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on Current Procedural Terminology dataset. --- layout: model title: Pipeline to Detect details of cellular structures (biobert) author: John Snow Labs name: ner_cellular_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_cellular_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_cellular_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cellular_biobert_pipeline_en_4.3.0_3.2_1679314449983.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cellular_biobert_pipeline_en_4.3.0_3.2_1679314449983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_cellular_biobert_pipeline", "en", "clinical/models") text = '''Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_cellular_biobert_pipeline", "en", "clinical/models") val text = "Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cellular_biobert.pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------------------------------|--------:|------:|:------------|-------------:| | 0 | intracellular signaling proteins | 27 | 58 | protein | 0.673333 | | 1 | human T-cell leukemia virus type 1 promoter | 130 | 172 | DNA | 0.426171 | | 2 | Tax | 186 | 188 | protein | 0.779 | | 3 | Tax-responsive element 1 | 193 | 216 | DNA | 0.756933 | | 4 | cyclic AMP-responsive members | 237 | 265 | protein | 0.629333 | | 5 | CREB/ATF family | 274 | 288 | protein | 0.8499 | | 6 | transcription factors | 293 | 313 | protein | 0.78165 | | 7 | Tax | 389 | 391 | protein | 0.8463 | | 8 | Tax-responsive element 1 | 431 | 454 | DNA | 0.713067 | | 9 | TRE-1 | 457 | 461 | DNA | 0.9983 | | 10 | lacZ gene | 582 | 590 | DNA | 0.7018 | | 11 | CYC1 promoter | 617 | 629 | DNA | 0.81865 | | 12 | TRE-1 | 663 | 667 | DNA | 0.9967 | | 13 | cyclic AMP response element-binding protein | 695 | 737 | protein | 0.51984 | | 14 | CREB | 740 | 743 | protein | 0.9708 | | 15 | CREB | 749 | 752 | protein | 0.8875 | | 16 | GAL4 activation domain | 767 | 788 | protein | 0.578633 | | 17 | GAD | 791 | 793 | protein | 0.6432 | | 18 | reporter gene | 848 | 860 | DNA | 0.61005 | | 19 | Tax | 863 | 865 | protein | 0.99 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cellular_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Sentence Entity Resolver for ICD10-PCS (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_icd10pcs date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-PCS codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts ICD10-PCS Codes and their normalized definitions. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10pcs_en_3.0.4_3.0_1621189710474.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10pcs_en_3.0.4_3.0_1621189710474.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_icd10pcs``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Procedure``` set in ```.setWhiteList()```.

## Results ```bash +--------------------+-----+---+---------+-------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+-------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM|DWY18ZZ| 0.0626|Hyperthermia of H...|DWY18ZZ:::6A3Z1ZZ...| |chronic renal ins...| 83|109| PROBLEM|DTY17ZZ| 0.0722|Contact Radiation...|DTY17ZZ:::04593ZZ...| | COPD| 113|116| PROBLEM|2W04X7Z| 0.0765|Change Intermitte...|2W04X7Z:::0J063ZZ...| | gastritis| 120|128| PROBLEM|04723Z6| 0.0826|Dilation of Gastr...|04723Z6:::04724Z6...| | TIA| 136|138| PROBLEM|00F5XZZ| 0.1074|Fragmentation in ...|00F5XZZ:::00F53ZZ...| |a non-ST elevatio...| 182|202| PROBLEM|B307ZZZ| 0.0750|Plain Radiography...|B307ZZZ:::2W59X3Z...| |Guaiac positive s...| 208|229| PROBLEM|3E1G38Z| 0.0886|Irrigation of Upp...|3E1G38Z:::3E1G38X...| |cardiac catheteri...| 295|317| TEST|4A0234Z| 0.0783|Measurement of Ca...|4A0234Z:::4A02X4A...| | PTCA| 324|327|TREATMENT|03SG3ZZ| 0.0507|Reposition Intrac...|03SG3ZZ:::0GCQ3ZZ...| | mid LAD lesion| 332|345| PROBLEM|02H73DZ| 0.0490|Insertion of Intr...|02H73DZ:::02163Z7...| +--------------------+-----+---+---------+-------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10pcs| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[icd10pcs_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on ICD10 Procedure Coding System dataset with ``sbiobert_base_cased_mli`` sentence embeddings. https://www.icd10data.com/ICD10PCS/Codes --- layout: model title: English T5ForConditionalGeneration Base Cased model (from nouamanetazi) author: John Snow Labs name: t5_cover_letter_base date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cover-letter-t5-base` is a English model originally trained by `nouamanetazi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_cover_letter_base_en_4.3.0_3.0_1675100617268.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_cover_letter_base_en_4.3.0_3.0_1675100617268.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_cover_letter_base","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_cover_letter_base","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_cover_letter_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|910.7 MB| ## References - https://huggingface.co/nouamanetazi/cover-letter-t5-base --- layout: model title: Fast Neural Machine Translation Model from Artificial languages to English author: John Snow Labs name: opus_mt_art_en date: 2021-06-01 tags: [open_source, seq2seq, translation, art, en, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: art target languages: en {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_art_en_xx_3.1.0_2.4_1622559545730.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_art_en_xx_3.1.0_2.4_1622559545730.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_art_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6_en_4.3.0_3.0_1674214482569.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6_en_4.3.0_3.0_1674214482569.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_16_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|416.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-6 --- layout: model title: Explain Document pipeline for Dutch (explain_document_lg) author: John Snow Labs name: explain_document_lg date: 2021-03-23 tags: [open_source, dutch, explain_document_lg, pipeline, nl] supported: true task: [Named Entity Recognition, Lemmatization] language: nl edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_lg_nl_3.0.0_3.0_1616513098571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_lg_nl_3.0.0_3.0_1616513098571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_lg', lang = 'nl') annotations = pipeline.fullAnnotate(""Hallo van John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_lg", lang = "nl") val result = pipeline.fullAnnotate("Hallo van John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hallo van John Snow Labs! ""] result_df = nlu.load('nl.explain.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-------------------------------|:------------------------------|:------------------------------------------|:------------------------------------------|:--------------------------------------------|:-----------------------------|:------------------------------------------|:-----------------------------| | 0 | ['Hallo van John Snow Labs! '] | ['Hallo van John Snow Labs!'] | ['Hallo', 'van', 'John', 'Snow', 'Labs!'] | ['Hallo', 'van', 'John', 'Snow', 'Labs!'] | ['PROPN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[-0.245989993214607,.,...]] | ['B-PER', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['Hallo', 'John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|nl| --- layout: model title: Legal Support Clause Binary Classifier author: John Snow Labs name: legclf_support_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `support` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `support` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_support_clause_en_1.0.0_3.2_1660123058175.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_support_clause_en_1.0.0_3.2_1660123058175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_support_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[support]| |[other]| |[other]| |[support]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_support_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.99 0.98 120 support 0.97 0.89 0.93 35 accuracy - - 0.97 155 macro-avg 0.97 0.94 0.95 155 weighted-avg 0.97 0.97 0.97 155 ``` --- layout: model title: Legal Terminations Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_terminations_bert date: 2023-03-05 tags: [en, legal, classification, clauses, terminations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Terminations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Terminations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_terminations_bert_en_1.0.0_3.0_1678050545103.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_terminations_bert_en_1.0.0_3.0_1678050545103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_terminations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Terminations]| |[Other]| |[Other]| |[Terminations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_terminations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.96 0.89 0.93 160 Terminations 0.88 0.95 0.91 129 accuracy - - 0.92 289 macro-avg 0.92 0.92 0.92 289 weighted-avg 0.92 0.92 0.92 289 ``` --- layout: model title: SNOMED Sentence Resolver (Spanish) author: John Snow Labs name: robertaresolve_snomed date: 2021-11-03 tags: [embeddings, es, snomed, entity_resolution, clinical, licensed] task: Entity Resolution language: es edition: Healthcare NLP 3.3.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps predetected ner chunks (from a `MedicalNerModel`, a `ChunkConverter` and a `Chunk2Doc`) to SNOMED terms and codes for the Spanish version of SNOMED. It requires Roberta Clinical Word Embeddings (`roberta_base_biomedical_es`) averaged with `SentenceEmbeddings`. ## Predicted Entities `SNOMED codes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/robertaresolve_snomed_es_3.3.0_3.0_1635933551478.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/robertaresolve_snomed_es_3.3.0_3.0_1635933551478.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use any `MedicalNer` Model from our ModelsHub that detects, for example, diagnosis, for Spanish. Then, use a `NerConverter` (in case your model has B-I-O notation). Create documents using `Chunk2Doc`. Then use a `Tokenizer` the split the chunk, and finally use the `roberta_base_biomedical_es` Roberta Embeddings model and a `SentenceEmbeddings` annotator with an average pooling strategy, as in the example.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... c2doc = nlp.Chunk2Doc() \ .setInputCols("ner_chunk") \ .setOutputCol("sentence") chunk_tokenizer = nlp.Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") chunk_word_embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner_chunk_word_embeddings") chunk_embeddings = nlp.SentenceEmbeddings() \ .setInputCols(["sentence", "ner_chunk_word_embeddings"]) \ .setOutputCol("ner_chunk_embeddings") \ .setPoolingStrategy("AVERAGE") er = medical.SentenceEntityResolverModel.pretrained("robertaresolve_snomed", "es", "clinical/models")\ .setInputCols(["sentence", "ner_chunk_embeddings"]) \ .setOutputCol("snomed_code") \ .setDistanceFunction("EUCLIDEAN") snomed_resolve_pipeline = Pipeline(stages = [ c2doc, chunk_tokenizer, chunk_word_embeddings, chunk_embeddings, er ]) empty = spark.createDataFrame([['']]).toDF("text") p_model = snomed_resolve_pipeline.fit(empty) test_sentence = "Mujer de 28 años con antecedentes de diabetes mellitus gestacional diagnosticada ocho años antes de la presentación y posterior diabetes mellitus tipo dos (DM2), un episodio previo de pancreatitis inducida por HTG tres años antes de la presentación, asociado con una hepatitis aguda, y obesidad con un índice de masa corporal (IMC) de 33,5 kg / m2, que se presentó con antecedentes de una semana de poliuria, polidipsia, falta de apetito y vómitos. Dos semanas antes de la presentación, fue tratada con un ciclo de cinco días de amoxicilina por una infección del tracto respiratorio. Estaba tomando metformina, glipizida y dapagliflozina para la DM2 y atorvastatina y gemfibrozil para la HTG. Había estado tomando dapagliflozina durante seis meses en el momento de la presentación. El examen físico al momento de la presentación fue significativo para la mucosa oral seca; significativamente, su examen abdominal fue benigno sin dolor a la palpación, protección o rigidez. Los hallazgos de laboratorio pertinentes al ingreso fueron: glucosa sérica 111 mg / dl, bicarbonato 18 mmol / l, anión gap 20, creatinina 0,4 mg / dl, triglicéridos 508 mg / dl, colesterol total 122 mg / dl, hemoglobina glucosilada (HbA1c) 10%. y pH venoso 7,27. La lipasa sérica fue normal a 43 U / L. Los niveles séricos de acetona no pudieron evaluarse ya que las muestras de sangre se mantuvieron hemolizadas debido a una lipemia significativa. La paciente ingresó inicialmente por cetosis por inanición, ya que refirió una ingesta oral deficiente durante los tres días previos a la admisión. Sin embargo, la química sérica obtenida seis horas después de la presentación reveló que su glucosa era de 186 mg / dL, la brecha aniónica todavía estaba elevada a 21, el bicarbonato sérico era de 16 mmol / L, el nivel de triglicéridos alcanzó un máximo de 2050 mg / dL y la lipasa fue de 52 U / L. Se obtuvo el nivel de β-hidroxibutirato y se encontró que estaba elevado a 5,29 mmol / L; la muestra original se centrifugó y la capa de quilomicrones se eliminó antes del análisis debido a la interferencia de la turbidez causada por la lipemia nuevamente. El paciente fue tratado con un goteo de insulina para euDKA y HTG con una reducción de la brecha aniónica a 13 y triglicéridos a 1400 mg / dL, dentro de las 24 horas. Se pensó que su euDKA fue precipitada por su infección del tracto respiratorio en el contexto del uso del inhibidor de SGLT2. La paciente fue atendida por el servicio de endocrinología y fue dada de alta con 40 unidades de insulina glargina por la noche, 12 unidades de insulina lispro con las comidas y metformina 1000 mg dos veces al día. Se determinó que todos los inhibidores de SGLT2 deben suspenderse indefinidamente. Tuvo un seguimiento estrecho con endocrinología post alta." result = p_model.transform(spark.createDataFrame(pd.DataFrame({'text': [test_sentence]}))) ``` ```scala ... val c2doc = new Chunk2Doc() .setInputCols(Array("ner_chunk")) .setOutputCol("sentence") val chunk_tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val chunk_word_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("ner_chunk_word_embeddings") val chunk_embeddings = new SentenceEmbeddings() .setInputCols(Array("sentence", "ner_chunk_word_embeddings")) .setOutputCol("ner_chunk_embeddings") .setPoolingStrategy("AVERAGE") val er = SentenceEntityResolverModel.pretrained("robertaresolve_snomed", "es", "clinical/models") .setInputCols(Array("ner_chunk_embeddings")) .setOutputCol("snomed_code") .setDistanceFunction("EUCLIDEAN") val snomed_pipeline = new Pipeline().setStages(Array( c2doc, chunk_tokenizer, chunk_word_embeddings, chunk_embeddings, er)) val test_sentence = 'Mujer de 28 años con antecedentes de diabetes mellitus gestacional diagnosticada ocho años antes de la presentación y posterior diabetes mellitus tipo dos (DM2), un episodio previo de pancreatitis inducida por HTG tres años antes de la presentación, asociado con una hepatitis aguda, y obesidad con un índice de masa corporal (IMC) de 33,5 kg / m2, que se presentó con antecedentes de una semana de poliuria, polidipsia, falta de apetito y vómitos. Dos semanas antes de la presentación, fue tratada con un ciclo de cinco días de amoxicilina por una infección del tracto respiratorio. Estaba tomando metformina, glipizida y dapagliflozina para la DM2 y atorvastatina y gemfibrozil para la HTG. Había estado tomando dapagliflozina durante seis meses en el momento de la presentación. El examen físico al momento de la presentación fue significativo para la mucosa oral seca; significativamente, su examen abdominal fue benigno sin dolor a la palpación, protección o rigidez. Los hallazgos de laboratorio pertinentes al ingreso fueron: glucosa sérica 111 mg / dl, bicarbonato 18 mmol / l, anión gap 20, creatinina 0,4 mg / dl, triglicéridos 508 mg / dl, colesterol total 122 mg / dl, hemoglobina glucosilada (HbA1c) 10%. y pH venoso 7,27. La lipasa sérica fue normal a 43 U / L. Los niveles séricos de acetona no pudieron evaluarse ya que las muestras de sangre se mantuvieron hemolizadas debido a una lipemia significativa. La paciente ingresó inicialmente por cetosis por inanición, ya que refirió una ingesta oral deficiente durante los tres días previos a la admisión. Sin embargo, la química sérica obtenida seis horas después de la presentación reveló que su glucosa era de 186 mg / dL, la brecha aniónica todavía estaba elevada a 21, el bicarbonato sérico era de 16 mmol / L, el nivel de triglicéridos alcanzó un máximo de 2050 mg / dL y la lipasa fue de 52 U / L. Se obtuvo el nivel de β-hidroxibutirato y se encontró que estaba elevado a 5,29 mmol / L; la muestra original se centrifugó y la capa de quilomicrones se eliminó antes del análisis debido a la interferencia de la turbidez causada por la lipemia nuevamente. El paciente fue tratado con un goteo de insulina para euDKA y HTG con una reducción de la brecha aniónica a 13 y triglicéridos a 1400 mg / dL, dentro de las 24 horas. Se pensó que su euDKA fue precipitada por su infección del tracto respiratorio en el contexto del uso del inhibidor de SGLT2. La paciente fue atendida por el servicio de endocrinología y fue dada de alta con 40 unidades de insulina glargina por la noche, 12 unidades de insulina lispro con las comidas y metformina 1000 mg dos veces al día. Se determinó que todos los inhibidores de SGLT2 deben suspenderse indefinidamente. Tuvo un seguimiento estrecho con endocrinología post alta.' val data = Seq(test_sentence).toDF("text") val result = snomed_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.resolve.snomed").predict("""Mujer de 28 años con antecedentes de diabetes mellitus gestacional diagnosticada ocho años antes de la presentación y posterior diabetes mellitus tipo dos (DM2), un episodio previo de pancreatitis inducida por HTG tres años antes de la presentación, asociado con una hepatitis aguda, y obesidad con un índice de masa corporal (IMC) de 33,5 kg / m2, que se presentó con antecedentes de una semana de poliuria, polidipsia, falta de apetito y vómitos. Dos semanas antes de la presentación, fue tratada con un ciclo de cinco días de amoxicilina por una infección del tracto respiratorio. Estaba tomando metformina, glipizida y dapagliflozina para la DM2 y atorvastatina y gemfibrozil para la HTG. Había estado tomando dapagliflozina durante seis meses en el momento de la presentación. El examen físico al momento de la presentación fue significativo para la mucosa oral seca; significativamente, su examen abdominal fue benigno sin dolor a la palpación, protección o rigidez. Los hallazgos de laboratorio pertinentes al ingreso fueron: glucosa sérica 111 mg / dl, bicarbonato 18 mmol / l, anión gap 20, creatinina 0,4 mg / dl, triglicéridos 508 mg / dl, colesterol total 122 mg / dl, hemoglobina glucosilada (HbA1c) 10%. y pH venoso 7,27. La lipasa sérica fue normal a 43 U / L. Los niveles séricos de acetona no pudieron evaluarse ya que las muestras de sangre se mantuvieron hemolizadas debido a una lipemia significativa. La paciente ingresó inicialmente por cetosis por inanición, ya que refirió una ingesta oral deficiente durante los tres días previos a la admisión. Sin embargo, la química sérica obtenida seis horas después de la presentación reveló que su glucosa era de 186 mg / dL, la brecha aniónica todavía estaba elevada a 21, el bicarbonato sérico era de 16 mmol / L, el nivel de triglicéridos alcanzó un máximo de 2050 mg / dL y la lipasa fue de 52 U / L. Se obtuvo el nivel de β-hidroxibutirato y se encontró que estaba elevado a 5,29 mmol / L; la muestra original se centrifugó y la capa de quilomicrones se eliminó antes del análisis debido a la interferencia de la turbidez causada por la lipemia nuevamente. El paciente fue tratado con un goteo de insulina para euDKA y HTG con una reducción de la brecha aniónica a 13 y triglicéridos a 1400 mg / dL, dentro de las 24 horas. Se pensó que su euDKA fue precipitada por su infección del tracto respiratorio en el contexto del uso del inhibidor de SGLT2. La paciente fue atendida por el servicio de endocrinología y fue dada de alta con 40 unidades de insulina glargina por la noche, 12 unidades de insulina lispro con las comidas y metformina 1000 mg dos veces al día. Se determinó que todos los inhibidores de SGLT2 deben suspenderse indefinidamente. Tuvo un seguimiento estrecho con endocrinología post alta.""") ```

## Results ```bash +----+-------------------------------+-------------+--------------+ | | ner_chunk | entity | snomed_code| |----+-------------------------------+-------------+--------------| | 0 | diabetes mellitus gestacional | DIAGNOSTICO | 11687002 | | 1 | diabetes mellitus tipo dos | DIAGNOSTICO | 44054006 | | 2 | pancreatitis | DIAGNOSTICO | 75694006 | | 3 | HTG | DIAGNOSTICO | 266569009 | | 4 | hepatitis aguda | DIAGNOSTICO | 37871000 | | 5 | obesidad | DIAGNOSTICO | 5476005 | | 6 | índice de masa corporal | DIAGNOSTICO | 162859006 | | 7 | poliuria | DIAGNOSTICO | 56574000 | | 8 | polidipsia | DIAGNOSTICO | 17173007 | | 9 | falta de apetito | DIAGNOSTICO | 49233005 | | 10 | vómitos | DIAGNOSTICO | 422400008 | | 11 | infección | DIAGNOSTICO | 40733004 | | 12 | HTG | DIAGNOSTICO | 266569009 | | 13 | dolor | DIAGNOSTICO | 22253000 | | 14 | rigidez | DIAGNOSTICO | 271587009 | | 15 | cetosis | DIAGNOSTICO | 2538008 | | 16 | infección | DIAGNOSTICO | 40733004 | +----+-------------------------------+-------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|robertaresolve_snomed| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk_doc, sentence_embeddings]| |Output Labels:|[snomed_code]| |Language:|es| |Case sensitive:|false| |Dependencies:|roberta_base_biomedical_es| --- layout: model title: Embeddings Healthcare author: John Snow Labs name: embeddings_healthcare class: WordEmbeddingsModel language: en nav_key: models repository: clinical/models date: 2020-03-26 task: Embeddings edition: Healthcare NLP 2.4.4 spark_version: 2.4 tags: [licensed,clinical,embeddings,en] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.h2_title} ## Data Source Trained on PubMed + ICD10 + UMLS + MIMIC III corpora https://www.nlm.nih.gov/databases/download/pubmed_medline.html {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_healthcare_en_2.4.4_2.4_1585188313964.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_healthcare_en_2.4.4_2.4_1585188313964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_healthcare","en","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_healthcare","en","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.glove.healthcare").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_healthcare``. {:.model-param} ## Model Information {:.table-model} |---------------|-----------------------| | Name: | embeddings_healthcare | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.4.4+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | en | | Dimension: | 400.0 | --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ThaisBeham) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_fira date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-fira` is a English model originally trained by `ThaisBeham`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_fira_en_4.3.0_3.0_1672767957708.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_fira_en_4.3.0_3.0_1672767957708.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_fira","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_fira","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_fira| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ThaisBeham/distilbert-base-uncased-finetuned-fira --- layout: model title: English Named Entity Recognition (from elastic) author: John Snow Labs name: distilbert_ner_distilbert_base_cased_finetuned_conll03_english date: 2022-05-16 tags: [distilbert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-finetuned-conll03-english` is a English model orginally trained by `elastic`. ## Predicted Entities `ORG`, `MISC`, `PER`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_cased_finetuned_conll03_english_en_3.4.2_3.0_1652721683253.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_cased_finetuned_conll03_english_en_3.4.2_3.0_1652721683253.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_distilbert_base_cased_finetuned_conll03_english| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/elastic/distilbert-base-cased-finetuned-conll03-english --- layout: model title: English DistilBertForQuestionAnswering model (from V3RX2000) author: John Snow Labs name: distilbert_qa_V3RX2000_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `V3RX2000`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_V3RX2000_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724840806.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_V3RX2000_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724840806.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_V3RX2000_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_V3RX2000_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_V3RX2000").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_V3RX2000_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/V3RX2000/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec2_large_960h_lv60_self TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_large_960h_lv60_self date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h_lv60_self` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_960h_lv60_self_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_lv60_self_en_4.2.0_3.0_1664036965244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_lv60_self_en_4.2.0_3.0_1664036965244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_960h_lv60_self", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_960h_lv60_self", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_960h_lv60_self| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|757.3 MB| --- layout: model title: Explain Document Pipeline - CARP author: John Snow Labs name: explain_clinical_doc_carp date: 2020-08-19 task: [Named Entity Recognition, Assertion Status, Relation Extraction, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 2.5.5 spark_version: 2.4 tags: [pipeline, en, clinical, licensed] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained pipeline with ``ner_clinical``, ``assertion_dl``, ``re_clinical`` and ``ner_posology``. It will extract clinical and medication entities, assign assertion status and find relationships between clinical entities. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_carp_en_2.5.5_2.4_1597841630062.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_carp_en_2.5.5_2.4_1597841630062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python carp_pipeline = PretrainedPipeline("explain_clinical_doc_carp","en","clinical/models") annotations = carp_pipeline.fullAnnotate("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting. She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals.""")[0] annotations.keys() ``` ```scala val carp_pipeline = new PretrainedPipeline("explain_clinical_doc_carp","en","clinical/models") val result = carp_pipeline.fullAnnotate("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting. She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals.""")(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.carp").predict("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting. She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals.""") ```

{:.h2_title} ## Results This pretrained pipeline gives the result of `ner_clinical`, `re_clinical`, `ner_posology` and `assertion_dl` models. ```bash | | chunks | ner_clinical | assertion | posology_chunk | ner_posology | relations | |---|-------------------------------|--------------|-----------|------------------|--------------|-----------| | 0 | gestational diabetes mellitus | PROBLEM | present | metformin | Drug | TrAP | | 1 | metformin | TREATMENT | present | 1000 mg | Strength | TrCP | | 2 | polyuria | PROBLEM | present | two times a day | Frequency | TrCP | | 3 | polydipsia | PROBLEM | present | 40 units | Dosage | TrWP | | 4 | poor appetite | PROBLEM | present | insulin glargine | Drug | TrCP | | 5 | vomiting | PROBLEM | present | at night | Frequency | TrAP | | 6 | insulin glargine | TREATMENT | present | 12 units | Dosage | TrAP | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_carp| |Type:|pipeline| |Compatibility:|Spark NLP 2.5.5| |License:|Licensed| |Edition:|Official| |Language:|[en]| {:.h2_title} ## Included Models - ner_clinical - assertion_dl - re_clinical - ner_posology --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from moghis) author: John Snow Labs name: xlmroberta_ner_base_finetuned_panx date: 2022-08-14 tags: [de, fr, open_source, xlm_roberta, ner, xx] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-fr-de` is a Multilingual model originally trained by `moghis`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_panx_xx_4.1.0_3.0_1660445027313.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_panx_xx_4.1.0_3.0_1660445027313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_panx","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_panx","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|858.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/moghis/xlm-roberta-base-finetuned-panx-fr-de --- layout: model title: Translate English to Germanic languages Pipeline author: John Snow Labs name: translate_en_gem date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, gem, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `gem` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_gem_xx_2.7.0_2.4_1609688259531.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_gem_xx_2.7.0_2.4_1609688259531.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_gem", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_gem", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.gem').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_gem| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Finnish BERT Embeddings (Base Cased) author: John Snow Labs name: bert_base_finnish_cased date: 2022-01-03 tags: [open_source, embeddings, fi, bert] task: Embeddings language: fi edition: Spark NLP 3.3.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A version of Google's BERT deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. FinBERT features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words than e.g. the previously released multilingual BERT models from Google. FinBERT has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train FinBERT. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_finnish_cased_fi_3.3.4_2.4_1641223279447.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_finnish_cased_fi_3.3.4_2.4_1641223279447.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_base_finnish_cased", "fi") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") sample_data= spark.createDataFrame([['Syväoppiminen perustuu keinotekoisiin hermoihin, jotka muodostavat monikerroksisen neuroverkon.']], ["text"]) nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(sample_data) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_base_finnish_cased", "fi") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("Syväoppiminen perustuu keinotekoisiin hermoihin, jotka muodostavat monikerroksisen neuroverkon.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("fi.embed_sentence.bert.cased").predict("""Syväoppiminen perustuu keinotekoisiin hermoihin, jotka muodostavat monikerroksisen neuroverkon.""") ```

## Results ```bash +--------------------+---------------+ | embeddings| token| +--------------------+---------------+ |[0.53366333, -0.4...| Syväoppiminen| |[0.49171034, -1.1...| perustuu| |[-0.0017492473, -...| keinotekoisiin| |[0.61259747, -0.7...| hermoihin| |[-0.008151092, -0...| ,| |[-0.4050159, -0.2...| jotka| |[-0.69079936, 0.6...| muodostavat| |[-0.45641452, 0.4...|monikerroksisen| |[1.278124, -1.218...| neuroverkon| |[0.42451048, -1.2...| .| +--------------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_finnish_cased| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert]| |Language:|fi| |Size:|464.2 MB| |Case sensitive:|true| --- layout: model title: Translate West Germanic languages to English Pipeline author: John Snow Labs name: translate_gmw_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, gmw, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `gmw` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_gmw_en_xx_2.7.0_2.4_1609685887934.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_gmw_en_xx_2.7.0_2.4_1609685887934.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gmw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gmw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gmw.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gmw_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223448939.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223448939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: English BertForMaskedLM Large Cased model author: John Snow Labs name: bert_embeddings_large_cased date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-cased` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_cased_en_4.2.4_3.0_1670020019196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_cased_en_4.2.4_3.0_1670020019196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_cased","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_cased","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/bert-large-cased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English BertForQuestionAnswering model (from bdickson) author: John Snow Labs name: bert_qa_bdickson_bert_base_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad` is a English model orginally trained by `bdickson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bdickson_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181090164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bdickson_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181090164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bdickson_bert_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bdickson_bert_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_bdickson").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bdickson_bert_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/bdickson/bert-base-uncased-finetuned-squad --- layout: model title: COVID BERT Embeddings (Large Uncased) author: John Snow Labs name: covidbert_large_uncased date: 2020-08-27 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BERT-large-uncased model, pretrained on a corpus of messages from Twitter about COVID-19. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/covidbert_large_uncased_en_2.6.0_2.4_1598484981419.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/covidbert_large_uncased_en_2.6.0_2.4_1598484981419.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("covidbert_large_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("covidbert_large_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.covidbert.large_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_covidbert_large_uncased_embeddings token [-1.934066891670227, 0.620597779750824, 0.0967... I [-0.5530431866645813, 1.1948248147964478, -0.0... love [0.255395770072937, 0.5808677077293396, 0.3073... NLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|covidbert_large_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/digitalepidemiologylab/covid-twitter-bert/2 --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-512-finetuned-squad-seed-0` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0_en_4.0.0_3.0_1654180865294.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0_en_4.0.0_3.0_1654180865294.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_512d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_few_shot_k_512_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-512-finetuned-squad-seed-0 --- layout: model title: Part of Speech for Persian author: John Snow Labs name: pos_ud_perdt date: 2020-11-30 task: Part of Speech Tagging language: fa edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [fa, pos] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_perdt_fa_2.7.0_2.4_1606724821106.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_perdt_fa_2.7.0_2.4_1606724821106.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_perdt", "fa") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate(["جان اسنو جدا از سلطنت شمال ، یک پزشک انگلیسی و رهبر توسعه بیهوشی و بهداشت پزشکی است."]) ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_perdt", "fa") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("جان اسنو جدا از سلطنت شمال ، یک پزشک انگلیسی و رهبر توسعه بیهوشی و بهداشت پزشکی است.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""جان اسنو جدا از سلطنت شمال ، یک پزشک انگلیسی و رهبر توسعه بیهوشی و بهداشت پزشکی است"""] pos_df = nlu.load('fa.pos').predict(text) pos_df ```

## Results ```bash {'pos': [Annotation(pos, 0, 2, NOUN, {'word': 'جان'}), Annotation(pos, 4, 7, NOUN, {'word': 'اسنو'}), Annotation(pos, 9, 11, ADJ, {'word': 'جدا'}), Annotation(pos, 13, 14, ADP, {'word': 'از'}), Annotation(pos, 16, 20, NOUN, {'word': 'سلطنت'}), Annotation(pos, 22, 25, NOUN, {'word': 'شمال'}), Annotation(pos, 27, 27, PUNCT, {'word': '،'}), Annotation(pos, 29, 30, NUM, {'word': 'یک'}), Annotation(pos, 32, 35, NOUN, {'word': 'پزشک'}), Annotation(pos, 37, 43, ADJ, {'word': 'انگلیسی'}), Annotation(pos, 45, 45, CCONJ, {'word': 'و'}), Annotation(pos, 47, 50, NOUN, {'word': 'رهبر'}), Annotation(pos, 52, 56, NOUN, {'word': 'توسعه'}), Annotation(pos, 58, 63, VERB, {'word': 'بیهوشی'}), Annotation(pos, 65, 65, CCONJ, {'word': 'و'}), Annotation(pos, 67, 72, NOUN, {'word': 'بهداشت'}), Annotation(pos, 74, 78, ADJ, {'word': 'پزشکی'}), Annotation(pos, 80, 82, AUX, {'word': 'است'}), Annotation(pos, 83, 83, PUNCT, {'word': '.'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_perdt| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[tags, document]| |Output Labels:|[pos]| |Language:|fa| ## Data Source The model is trained on data obtained from [https://universaldependencies.org](https://universaldependencies.org) ## Benchmarking ```bash | | | precision | recall | f1-score | support | |---:|:-------------|:------------|:---------|-----------:|----------:| | 0 | ADJ | 0.88 | 0.88 | 0.88 | 1647 | | 1 | ADP | 0.99 | 0.99 | 0.99 | 3402 | | 2 | ADV | 0.94 | 0.91 | 0.92 | 383 | | 3 | AUX | 0.99 | 0.99 | 0.99 | 1000 | | 4 | CCONJ | 1.00 | 1.00 | 1 | 1022 | | 5 | DET | 0.94 | 0.96 | 0.95 | 490 | | 6 | INTJ | 0.88 | 0.81 | 0.85 | 27 | | 7 | NOUN | 0.95 | 0.96 | 0.95 | 8201 | | 8 | NUM | 0.94 | 0.97 | 0.96 | 293 | | 9 | None | 1.00 | 0.99 | 0.99 | 289 | | 10 | PART | 1.00 | 0.86 | 0.92 | 28 | | 11 | PRON | 0.98 | 0.97 | 0.98 | 1117 | | 12 | PROPN | 0.84 | 0.78 | 0.81 | 1107 | | 13 | PUNCT | 1.00 | 1.00 | 1 | 2134 | | 14 | SCONJ | 0.98 | 0.98 | 0.98 | 630 | | 15 | VERB | 0.99 | 0.99 | 0.99 | 2581 | | 16 | accuracy | | | 0.96 | 24351 | | 17 | macro avg | 0.96 | 0.94 | 0.95 | 24351 | | 18 | weighted avg | 0.96 | 0.96 | 0.96 | 24351 | ``` --- layout: model title: Legal Rights Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_rights_agreement date: 2022-11-24 tags: [en, legal, classification, agreement, rights, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_rights_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `rights-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `rights-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_rights_agreement_en_1.0.0_3.0_1669294308500.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_rights_agreement_en_1.0.0_3.0_1669294308500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[rights-agreement]| |[other]| |[other]| |[rights-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_rights_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.98 0.98 90 rights-agreement 0.94 0.97 0.95 30 accuracy - - 0.97 120 macro-avg 0.96 0.97 0.97 120 weighted-avg 0.98 0.97 0.98 120 ``` --- layout: model title: Translate Korean to English Pipeline author: John Snow Labs name: translate_ko_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ko, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ko` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ko_en_xx_2.7.0_2.4_1609688668059.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ko_en_xx_2.7.0_2.4_1609688668059.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ko_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ko_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ko.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ko_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Vietnamese Deberta Embeddings model (from hieule) author: John Snow Labs name: deberta_embeddings_spm_vie date: 2023-03-12 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, vie, tensorflow] task: Embeddings language: vie edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spm-vie-deberta` is a Vietnamese model originally trained by `hieule`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_spm_vie_vie_4.3.1_3.0_1678627522214.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_spm_vie_vie_4.3.1_3.0_1678627522214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_spm_vie","vie") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_spm_vie","vie") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_spm_vie| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|vie| |Size:|290.3 MB| |Case sensitive:|false| ## References https://huggingface.co/hieule/spm-vie-deberta --- layout: model title: English asr_wav2vec2_large_xls_r_300m_hyAM_batch4 TFWav2Vec2ForCTC from lilitket author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_hyAM_batch4` is a English model originally trained by lilitket. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_en_4.2.0_3.0_1664119468999.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_en_4.2.0_3.0_1664119468999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Second Supplemental Indenture Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_second_supplemental_indenture_bert date: 2023-02-02 tags: [en, legal, classification, second, supplemental, indenture, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_second_supplemental_indenture_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `second-supplemental-indenture` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `second-supplemental-indenture`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_second_supplemental_indenture_bert_en_1.0.0_3.0_1675359737104.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_second_supplemental_indenture_bert_en_1.0.0_3.0_1675359737104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_second_supplemental_indenture_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[second-supplemental-indenture]| |[other]| |[other]| |[second-supplemental-indenture]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_second_supplemental_indenture_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.97 0.96 73 second-supplemental-indenture 0.95 0.90 0.92 39 accuracy - - 0.95 112 macro-avg 0.95 0.94 0.94 112 weighted-avg 0.95 0.95 0.95 112 ``` --- layout: model title: Legal Material contracts Clause Binary Classifier author: John Snow Labs name: legclf_material_contracts_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `material-contracts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `material-contracts` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_material_contracts_clause_en_1.0.0_3.2_1660122646574.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_material_contracts_clause_en_1.0.0_3.2_1660122646574.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_material_contracts_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[material-contracts]| |[other]| |[other]| |[material-contracts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_material_contracts_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support material-contracts 0.85 0.79 0.82 29 other 0.94 0.96 0.95 93 accuracy - - 0.92 122 macro-avg 0.89 0.88 0.88 122 weighted-avg 0.92 0.92 0.92 122 ``` --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-64-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2_en_4.0.0_3.0_1657185416907.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2_en_4.0.0_3.0_1657185416907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-64-finetuned-squad-seed-2 --- layout: model title: Word2Vec Embeddings in Sanskrit (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, sa, open_source] task: Embeddings language: sa edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sa_3.4.1_3.0_1647455309990.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sa_3.4.1_3.0_1647455309990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") embeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") classsifier_dl = nlp.ClassifierDLModel.pretrained("finclf_bank_complaints", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("label")\ clf_pipeline = nlp.Pipeline( stages = [ document_assembler, embeddings, classsifier_dl ]) light_pipeline = LightPipeline(clf_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.annotate("""Over the course of 30 days I have filed a dispute in regards to inaccurate and false information on my credit report. Ive attached a copy of my dispute mailed in certified to Equifax and they are still reporting these incorrect items. According to the fair credit ACT, section 609 ( a ) ( 1 ) ( A ) they are required by Federal Law to only report Accurate information and the have not done so. They have not provided me with any proof i.e. and original consumer contract with my signature on it proving that this is my account.Further more, I would like to make a formal complaint that Ive tried calling Equifax Over 10 times this week and every single time Ive called Ive asked for a representative in the fraud dispute department wants transfer it over there when you speak to the representative and let them know that you are looking to dispute inquiries and accounts due to fraud they immediately transfer you to their survey line essentially ending the call. I believe Equifax is training their representatives to not help consumers over the phone and performing unethical practices. Once I finally got a hold of a representative she told me that she could not help because I did not send in my Social Security card which violates my consumer rights. So Im Making a formal CFPB complaint that you will correct Equifaxs actions. Below Ive written what is also included in the files uploaded, my disputes for inaccuracies on my credit report.""") result['label'] ```

## Results ```bash ['Credit Reporting'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_bank_complaints| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References https://www.consumerfinance.gov/data-research/consumer-complaints/#download-the-data ## Benchmarking ```bash label precision recall f1-score support Accounts 0.77 0.73 0.75 490 Credit_Cards 0.75 0.68 0.72 461 Credit_Reporting 0.73 0.81 0.76 488 Debt_Collection 0.72 0.72 0.72 459 Loans 0.78 0.78 0.78 472 Money_Transfer_and_Currency 0.82 0.84 0.83 482 Mortgage 0.87 0.87 0.87 488 accuracy - - 0.78 3340 macro-avg 0.78 0.78 0.78 3340 weighted-avg 0.78 0.78 0.78 3340 ``` --- layout: model title: Spanish BertForQuestionAnswering model (from MMG) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG date: 2022-06-03 tags: [es, open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-sqac-finetuned-squad2-es` is a Spanish model orginally trained by `MMG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG_es_4.0.0_3.0_1654249736195.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG_es_4.0.0_3.0_1654249736195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2_sqac.bert.base_cased.by_MMG").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad2_es_MMG| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MMG/bert-base-spanish-wwm-cased-finetuned-sqac-finetuned-squad2-es --- layout: model title: SDOH Substance Usage For Binary Classification author: John Snow Labs name: genericclassifier_sdoh_substance_usage_binary_sbiobert_cased_mli date: 2023-01-14 tags: [en, licensed, generic_classifier, sdoh, substance, clinical] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting substance use in clinical notes and trained by using GenericClassifierApproach annotator. `Present:` if the patient was a current consumer of substance or the patient was a consumer in the past and had quit or if the patient had never consumed substance. `None:` if there was no related text. ## Predicted Entities `Present`, `None` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_substance_usage_binary_sbiobert_cased_mli_en_4.2.4_3.0_1673697973649.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_substance_usage_binary_sbiobert_cased_mli_en_4.2.4_3.0_1673697973649.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+---------+ | text| result| +----------------------------------------------------------------------------------------------------+---------+ |Lives in apartment with 16-year-old daughter. Denies EtOH use currently although reports occasion...|[Present]| |The patient quit smoking approximately two years ago with an approximately a 40 pack year history...| [None]| +----------------------------------------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_substance_usage_binary_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| ## Benchmarking ```bash label precision recall f1-score support None 0.91 0.83 0.87 898 Present 0.76 0.87 0.81 540 accuracy - - 0.85 1438 macro-avg 0.83 0.85 0.84 1438 weighted-avg 0.85 0.85 0.85 1438 ``` --- layout: model title: Legal Multilabel Classifier on Covid-19 Exceptions (Italian) author: John Snow Labs name: legmulticlf_covid19_exceptions_italian date: 2023-04-20 tags: [it, licensed, legal, multilabel, classification, tensorflow] task: Text Classification language: it edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is the Multi-Label Text Classification model that can be used to identify up to 5 classes to facilitate analysis, discovery, and comparison of legal texts in Italian related to COVID-19 exception measures. The classes are as follows: - Closures/lockdown - Government_oversight - Restrictions_of_daily_liberties - Restrictions_of_fundamental_rights_and_civil_liberties - State_of_Emergency ## Predicted Entities `Closures/lockdown`, `Government_oversight`, `Restrictions_of_daily_liberties`, `Restrictions_of_fundamental_rights_and_civil_liberties`, `State_of_Emergency` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legmulticlf_covid19_exceptions_italian_it_1.0.0_3.0_1681985472330.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legmulticlf_covid19_exceptions_italian_it_1.0.0_3.0_1681985472330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_12", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_12", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_12| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English asr_hausa_4_wa2vec_data_aug_xls_r_300m TFWav2Vec2ForCTC from Tiamz author: John Snow Labs name: pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_hausa_4_wa2vec_data_aug_xls_r_300m` is a English model originally trained by Tiamz. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m_en_4.2.0_3.0_1664108237641.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m_en_4.2.0_3.0_1664108237641.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_hausa_4_wa2vec_data_aug_xls_r_300m| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Warranty NER (md) author: John Snow Labs name: legner_warranty_md date: 2022-12-01 tags: [warranty, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_warranty_clause` Text Classifier to select only these paragraphs; This is a Legal Named Entity Recognition Model to identify the Subject (who), Action (what), Object(the indemnification) and Indirect Object (to whom) from Warranty clauses. This is a `md` (medium version) of the classifier, trained with more data and being more resistent to false positives outside the specific section, which may help to run it at whole document level (although not recommended). ## Predicted Entities `WARRANTY`, `WARRANTY_ACTION`, `WARRANTY_SUBJECT`, `WARRANTY_INDIRECT_OBJECT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_warranty_md_en_1.0.0_3.0_1669893390077.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_warranty_md_en_1.0.0_3.0_1669893390077.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------------------+------------------------+ |chunk |entity | +--------------------------------------------------------------------------+------------------------+ |SONY |WARRANTY_SUBJECT | |makes the following representations and warranties |WARRANTY_ACTION | |PURCHASER |WARRANTY_INDIRECT_OBJECT| |shall be true and correct as of the date hereof and as of the Closing Date|WARRANTY | |shall be unaffected by any investigation |WARRANTY | |SONY |WARRANTY_SUBJECT | |has the right and power to enter into this IP Agreement |WARRANTY | +--------------------------------------------------------------------------+------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_warranty_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.1 MB| ## References In-house annotated examples from CUAD legal dataset ## Benchmarking ```bash label tp fp fn prec rec f1 I-WARRANTY_SUBJECT 23 9 19 0.71875 0.54761904 0.62162155 B-WARRANTY 111 36 34 0.75510204 0.76551723 0.760274 B-WARRANTY_SUBJECT 55 31 33 0.6395349 0.625 0.6321839 I-WARRANTY_INDIRECT_OBJECT 18 6 3 0.75 0.85714287 0.79999995 I-WARRANTY_ACTION 77 8 14 0.90588236 0.84615386 0.875 B-WARRANTY_ACTION 36 4 4 0.9 0.9 0.9 I-WARRANTY 1686 487 313 0.7758859 0.8434217 0.8082455 B-WARRANTY_INDIRECT_OBJECT 34 12 6 0.73913044 0.85 0.79069775 Macro-average 2040 593 426 0.7730357 0.7793569 0.7761834 Micro-average 2040 593 426 0.77478164 0.8272506 0.80015695 ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from niklaspm) author: John Snow Labs name: bert_qa_linkbert_base_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `linkbert-base-finetuned-squad` is a English model originally trained by `niklaspm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_linkbert_base_finetuned_squad_en_4.0.0_3.0_1657189758932.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_linkbert_base_finetuned_squad_en_4.0.0_3.0_1657189758932.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_linkbert_base_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_linkbert_base_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_linkbert_base_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/niklaspm/linkbert-base-finetuned-squad - https://arxiv.org/abs/2203.15827 --- layout: model title: Translate English to Tok Pisin Pipeline author: John Snow Labs name: translate_en_tpi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tpi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tpi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tpi_xx_2.7.0_2.4_1609686751040.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tpi_xx_2.7.0_2.4_1609686751040.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tpi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tpi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tpi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tpi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Longformer (Base, 4096) author: John Snow Labs name: legal_longformer_base date: 2022-10-20 tags: [en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.1 spark_version: [3.2, 3.0] supported: true annotator: LongformerEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [Longformer](https://arxiv.org/abs/2004.05150) is a transformer model for long documents. `legal_longformer_base` is a BERT-like model started from the RoBERTa checkpoint and pretrained for MLM on long documents. It supports sequences of length up to 4,096 and it's specifically trained on *legal documents* Longformer uses a combination of a sliding window (local) attention and global attention. Global attention is user-configured based on the task to allow the model to learn task-specific representations. If you use `Longformer` in your research, please cite [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150). ``` @article{Beltagy2020Longformer, title={Longformer: The Long-Document Transformer}, author={Iz Beltagy and Matthew E. Peters and Arman Cohan}, journal={arXiv:2004.05150}, year={2020}, } ``` `Longformer` is an open-source project developed by [the Allen Institute for Artificial Intelligence (AI2)](http://www.allenai.org). AI2 is a non-profit institute with the mission to contribute to humanity through high-impact AI research and engineering. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/legal_longformer_base_en_4.2.1_3.2_1666282710556.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/legal_longformer_base_en_4.2.1_3.2_1666282710556.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = LongformerEmbeddings\ .pretrained("legal_longformer_base", "en")\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings")\ .setCaseSensitive(True)\ .setMaxSentenceLength(4096) ``` ```scala val embeddings = LongformerEmbeddings.pretrained("legal_longformer_base", "en") .setInputCols("document", "token") .setOutputCol("embeddings") .setCaseSensitive(true) .setMaxSentenceLength(4096) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legal_longformer_base| |Compatibility:|Spark NLP 4.2.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|531.1 MB| |Case sensitive:|true| |Max sentence length:|4096| ## References https://huggingface.co/saibo/legal-longformer-base-4096 --- layout: model title: Pipeline to Detect Genetic Cancer Entities author: John Snow Labs name: ner_cancer_genetics_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_cancer_genetics](https://nlp.johnsnowlabs.com/2021/03/31/ner_cancer_genetics_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_pipeline_en_4.3.0_3.2_1678864026558.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_pipeline_en_4.3.0_3.2_1678864026558.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_cancer_genetics_pipeline", "en", "clinical/models") text = '''The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_cancer_genetics_pipeline", "en", "clinical/models") val text = "The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cancer_genetics.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------------------------------------------------|--------:|------:|:------------|-------------:| | 0 | human KCNJ9 | 4 | 14 | protein | 0.674 | | 1 | Kir 3.3 | 17 | 23 | protein | 0.95355 | | 2 | GIRK3 | 26 | 30 | protein | 0.5127 | | 3 | G-protein-activated inwardly rectifying potassium (GIRK) channel family | 52 | 122 | protein | 0.691744 | | 4 | KCNJ9 locus | 173 | 183 | DNA | 0.97875 | | 5 | chromosome 1q21-23 | 188 | 205 | DNA | 0.95305 | | 6 | coding exons | 357 | 368 | DNA | 0.63345 | | 7 | identified14 single nucleotide polymorphisms | 451 | 494 | DNA | 0.6994 | | 8 | SNPs), | 497 | 502 | DNA | 0.79075 | | 9 | KCNJ9 gene | 801 | 810 | DNA | 0.95605 | | 10 | KCNJ9 protein | 868 | 880 | protein | 0.844 | | 11 | locus | 931 | 935 | DNA | 0.9685 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cancer_genetics_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_12_h_256 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-12_H-256` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_256_zh_4.2.4_3.0_1670021539975.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_256_zh_4.2.4_3.0_1670021539975.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_256","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_256","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_12_h_256| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|57.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-12_H-256 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Mapping RXNORM Codes with Their Corresponding UMLS Codes author: John Snow Labs name: rxnorm_umls_mapper date: 2022-06-26 tags: [rxnorm, umls, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps RXNORM codes to corresponding UMLS codes. ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapper_en_3.5.3_3.0_1656276292081.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapper_en_3.5.3_3.0_1656276292081.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli", "en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxnorm_augmented", "en", "clinical/models")\ .setInputCols(["ner_chunk", "sbert_embeddings"])\ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel\ .pretrained("rxnorm_umls_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("umls_mappings")\ .setRels(["umls_code"]) pipeline = Pipeline(stages = [ documentAssembler, sbert_embedder, rxnorm_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline= LightPipeline(model) result = light_pipeline.fullAnnotate("amlodipine 5 MG") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm_augmented", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel .pretrained("rxnorm_umls_mapper", "en", "clinical/models") .setInputCols("rxnorm_code") .setOutputCol("umls_mappings") .setRels(Array("umls_code")) val pipeline = new Pipeline(stages = Array( documentAssembler, sbert_embedder, rxnorm_resolver, chunkerMapper )) val data = Seq("amlodipine 5 MG").toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.rxnorm_to_umls").predict("""amlodipine 5 MG""") ```

## Results ```bash | | ner_chunk | rxnorm_code | umls_mappings | |---:|:----------------|--------------:|:----------------| | 0 | amlodipine 5 MG | 329528 | C1124796 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_umls_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[rxnorm_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|1.9 MB| ## References This pretrained model maps RXNORM codes to corresponding UMLS codes under the Unified Medical Language System (UMLS). --- layout: model title: Pipeline to Detect Clinical Entities (ner_eu_clinical_case - es) author: John Snow Labs name: ner_eu_clinical_case_pipeline date: 2023-03-08 tags: [es, clinical, licensed, ner] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_eu_clinical_case](https://nlp.johnsnowlabs.com/2023/02/01/ner_eu_clinical_case_es.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_pipeline_es_4.3.0_3.2_1678261388612.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_pipeline_es_4.3.0_3.2_1678261388612.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_eu_clinical_case_pipeline", "es", "clinical/models") text = " Un niño de 3 años con trastorno autista en el hospital de la sala pediátrica A del hospital universitario. No tiene antecedentes familiares de enfermedad o trastorno del espectro autista. El niño fue diagnosticado con un trastorno de comunicación severo, con dificultades de interacción social y retraso en el procesamiento sensorial. Los análisis de sangre fueron normales (hormona estimulante de la tiroides (TSH), hemoglobina, volumen corpuscular medio (MCV) y ferritina). La endoscopia alta también mostró un tumor submucoso que causaba una obstrucción subtotal de la salida gástrica. Ante la sospecha de tumor del estroma gastrointestinal, se realizó gastrectomía distal. El examen histopatológico reveló proliferación de células fusiformes en la capa submucosa. " result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_eu_clinical_case_pipeline", "es", "clinical/models") val text = " Un niño de 3 años con trastorno autista en el hospital de la sala pediátrica A del hospital universitario. No tiene antecedentes familiares de enfermedad o trastorno del espectro autista. El niño fue diagnosticado con un trastorno de comunicación severo, con dificultades de interacción social y retraso en el procesamiento sensorial. Los análisis de sangre fueron normales (hormona estimulante de la tiroides (TSH), hemoglobina, volumen corpuscular medio (MCV) y ferritina). La endoscopia alta también mostró un tumor submucoso que causaba una obstrucción subtotal de la salida gástrica. Ante la sospecha de tumor del estroma gastrointestinal, se realizó gastrectomía distal. El examen histopatológico reveló proliferación de células fusiformes en la capa submucosa. " val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:-----------------------------|--------:|------:|:-------------------|-------------:| | 0 | Un niño de 3 años | 1 | 17 | patient | 0.68856 | | 1 | trastorno | 23 | 31 | clinical_event | 0.9976 | | 2 | autista | 33 | 39 | clinical_condition | 0.7979 | | 3 | antecedentes | 117 | 128 | clinical_event | 0.7161 | | 4 | enfermedad | 144 | 153 | clinical_event | 0.5444 | | 5 | trastorno | 157 | 165 | clinical_event | 0.9914 | | 6 | del espectro autista | 167 | 186 | clinical_condition | 0.5385 | | 7 | El niño | 189 | 195 | patient | 0.87065 | | 8 | diagnosticado | 201 | 213 | clinical_event | 0.6442 | | 9 | trastorno | 222 | 230 | clinical_event | 0.836 | | 10 | de comunicación severo | 232 | 253 | clinical_condition | 0.501067 | | 11 | dificultades | 260 | 271 | clinical_event | 0.8807 | | 12 | retraso | 297 | 303 | clinical_event | 0.6975 | | 13 | análisis | 340 | 347 | clinical_event | 0.9664 | | 14 | sangre | 352 | 357 | bodypart | 0.9251 | | 15 | normales | 366 | 373 | units_measurements | 0.9838 | | 16 | hormona | 376 | 382 | clinical_event | 0.398 | | 17 | la tiroides | 399 | 409 | bodypart | 0.37665 | | 18 | TSH | 412 | 414 | clinical_event | 0.9389 | | 19 | hemoglobina | 418 | 428 | clinical_event | 0.2746 | | 20 | volumen | 431 | 437 | clinical_event | 0.9674 | | 21 | MCV | 458 | 460 | clinical_event | 0.6897 | | 22 | ferritina | 465 | 473 | clinical_event | 0.8188 | | 23 | endoscopia | 480 | 489 | clinical_event | 0.9953 | | 24 | mostró | 504 | 509 | clinical_event | 0.9998 | | 25 | tumor | 514 | 518 | clinical_event | 0.9866 | | 26 | submucoso | 520 | 528 | clinical_condition | 0.6053 | | 27 | obstrucción | 546 | 556 | clinical_event | 0.9974 | | 28 | tumor | 610 | 614 | clinical_event | 0.7284 | | 29 | del estroma gastrointestinal | 616 | 643 | bodypart | 0.577067 | | 30 | gastrectomía | 657 | 668 | clinical_event | 0.9666 | | 31 | examen | 681 | 686 | clinical_event | 0.9738 | | 32 | reveló | 704 | 709 | clinical_event | 0.9993 | | 33 | proliferación | 711 | 723 | clinical_event | 0.9996 | | 34 | células fusiformes | 728 | 745 | bodypart | 0.7001 | | 35 | la capa submucosa | 750 | 766 | bodypart | 0.641267 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_case_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English Named Entity Recognition (from sagorsarker) author: John Snow Labs name: bert_ner_codeswitch_nepeng_lid_lince date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `codeswitch-nepeng-lid-lince` is a English model orginally trained by `sagorsarker`. ## Predicted Entities `mixed`, `other`, `en`, `ambiguous`, `ne`, `nep` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_nepeng_lid_lince_en_3.4.2_3.0_1652097400616.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_nepeng_lid_lince_en_3.4.2_3.0_1652097400616.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_codeswitch_nepeng_lid_lince| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sagorsarker/codeswitch-nepeng-lid-lince - https://ritual.uh.edu/lince/home - https://github.com/sagorbrur/codeswitch --- layout: model title: Translate English to Indonesian Pipeline author: John Snow Labs name: translate_en_id date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, id, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `id` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_id_xx_2.7.0_2.4_1609690363498.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_id_xx_2.7.0_2.4_1609690363498.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_id", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_id", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.id').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_id| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German T5ForConditionalGeneration Cased model (from dehio) author: John Snow Labs name: t5_german_qg_e2e_quad date: 2023-01-30 tags: [de, open_source, t5, tensorflow] task: Text Generation language: de edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `german-qg-t5-e2e-quad` is a German model originally trained by `dehio`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_german_qg_e2e_quad_de_4.3.0_3.0_1675102645662.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_german_qg_e2e_quad_de_4.3.0_3.0_1675102645662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_german_qg_e2e_quad","de") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_german_qg_e2e_quad","de") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_german_qg_e2e_quad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|de| |Size:|924.3 MB| ## References - https://huggingface.co/dehio/german-qg-t5-e2e-quad --- layout: model title: Pipeline to Detect PHI in text (enriched-biobert) author: John Snow Labs name: ner_deid_enriched_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_enriched_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_enriched_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_pipeline_en_4.3.0_3.2_1679316429600.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_pipeline_en_4.3.0_3.2_1679316429600.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_enriched_biobert_pipeline", "en", "clinical/models") text = '''A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_enriched_biobert_pipeline", "en", "clinical/models") val text = "A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.ner_enriched_biobert.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:-------------|-------------:| | 0 | 2093-01-13 | 17 | 26 | DATE | 0.9267 | | 1 | David Hale | 29 | 38 | DOCTOR | 0.7949 | | 2 | Hendrickson, Ora | 53 | 68 | PATIENT | 0.637733 | | 3 | 7194334 | 76 | 82 | PHONE | 0.4939 | | 4 | Cocke County Baptist Hospital | 114 | 142 | HOSPITAL | 0.6199 | | 5 | 0295 Keats Street | 145 | 161 | STREET | 0.592433 | | 6 | 302) 786-5227 | 174 | 186 | PHONE | 0.846833 | | 7 | Brothers Coal-Mine | 253 | 270 | ORGANIZATION | 0.45085 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_enriched_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Chuvash RobertaForQuestionAnswering (from sunitha) author: John Snow Labs name: roberta_qa_CV_Custom_DS date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: cv edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `CV_Custom_DS` is a Chuvash model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_CV_Custom_DS_cv_4.0.0_3.0_1655726596821.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_CV_Custom_DS_cv_4.0.0_3.0_1655726596821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_CV_Custom_DS","cv") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_CV_Custom_DS","cv") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("cv.answer_question.roberta.cv_custom_ds.by_sunitha").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_CV_Custom_DS| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|cv| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/CV_Custom_DS --- layout: model title: English Deberta Embeddings model (from smeoni) author: John Snow Labs name: deberta_embeddings_nbme_V3_large date: 2023-03-13 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, en, tensorflow] task: Embeddings language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `nbme-deberta-V3-large` is a English model originally trained by `smeoni`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_nbme_V3_large_en_4.3.1_3.0_1678713648667.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_nbme_V3_large_en_4.3.1_3.0_1678713648667.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_nbme_V3_large","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_nbme_V3_large","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_nbme_V3_large| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|false| ## References https://huggingface.co/smeoni/nbme-deberta-V3-large --- layout: model title: English BertForMaskedLM Large Cased model (from VMware) author: John Snow Labs name: bert_embeddings_v_2021_large date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `vbert-2021-large` is a English model originally trained by `VMware`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_v_2021_large_en_4.2.4_3.0_1670023012204.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_v_2021_large_en_4.2.4_3.0_1670023012204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_v_2021_large","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_v_2021_large","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_v_2021_large| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/VMware/vbert-2021-large - https://medium.com/@rickbattle/weaknesses-of-wordpiece-tokenization-eb20e37fec99 --- layout: model title: English BertForQuestionAnswering Uncased model (from aodiniz) author: John Snow Labs name: bert_qa_uncased_l_2_h_128_a_2_squad2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-2_H-128_A-2_squad2` is a English model originally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_uncased_l_2_h_128_a_2_squad2_en_4.0.0_3.0_1657188893270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_uncased_l_2_h_128_a_2_squad2_en_4.0.0_3.0_1657188893270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_uncased_l_2_h_128_a_2_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_uncased_l_2_h_128_a_2_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_uncased_l_2_h_128_a_2_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|16.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-2_H-128_A-2_squad2 --- layout: model title: Detect Entities (GloVe) author: John Snow Labs name: ner_dl date: 2020-03-19 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.4.3 spark_version: 2.4 tags: [ner, en, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description `ner_dl` is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. It was trained on the CoNLL 2003 text corpus. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. `ner_dl` model is trained with GloVe 100D word embeddings, so be sure to use the same embeddings in the pipeline. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ner_dl_en_2.4.3_2.4_1584624950746.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ner_dl_en_2.4.3_2.4_1584624950746.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained('glove_100d') \ .setInputCols(["document", 'token']) \ .setOutputCol("embeddings") ner_model = NerDLModel.pretrained("ner_dl", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_100d") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("ner_dl", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.dl.glove.6B_100d').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates III |PER | |American |MISC | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |Born |LOC | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | |Gates |PER | |Gates |PER | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|ORG | |Melinda Gates |PER | |Ray Ozzie |PER | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_dl| |Type:|ner| |Compatibility:| Spark NLP 2.4.3+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from[CoNLL 2003 Data Set](https://github.com/synalp/NER/tree/master/corpus/CoNLL-2003) --- layout: model title: Korean Bert Embeddings author: John Snow Labs name: bert_embeddings_bert_base date: 2022-04-11 tags: [bert, embeddings, ko, open_source] task: Embeddings language: ko edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base` is a Korean model orginally trained by `klue`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_ko_3.4.2_3.0_1649675453798.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_ko_3.4.2_3.0_1649675453798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ko| |Size:|415.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/klue/bert-base - https://github.com/KLUE-benchmark/KLUE - https://arxiv.org/abs/2105.09680 --- layout: model title: Persian BertForQuestionAnswering model (from ForutanRad) author: John Snow Labs name: bert_qa_bert_fa_QA_v1 date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-fa-QA-v1` is a Persian model orginally trained by `ForutanRad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_fa_QA_v1_fa_4.0.0_3.0_1654181654761.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_fa_QA_v1_fa_4.0.0_3.0_1654181654761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_fa_QA_v1","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_fa_QA_v1","fa") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fa.answer_question.bert.by_ForutanRad").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_fa_QA_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ForutanRad/bert-fa-QA-v1 - https://arxiv.org/abs/2005.12515 --- layout: model title: Slovak RobertaForTokenClassification Cased model (from crabz) author: John Snow Labs name: roberta_token_classifier_slovakbert_ner date: 2023-03-01 tags: [sk, open_source, roberta, token_classification, ner, tensorflow] task: Named Entity Recognition language: sk edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `slovakbert-ner` is a Slovak model originally trained by `crabz`. ## Predicted Entities `4`, `2`, `6`, `1`, `0`, `5`, `3` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_slovakbert_ner_sk_4.3.0_3.0_1677703644531.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_slovakbert_ner_sk_4.3.0_3.0_1677703644531.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_slovakbert_ner","sk") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_slovakbert_ner","sk") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_slovakbert_ner| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sk| |Size:|439.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/crabz/slovakbert-ner - https://paperswithcode.com/sota?task=Token+Classification&dataset=wikiann --- layout: model title: Maltese Lemmatizer author: John Snow Labs name: lemma date: 2021-04-02 tags: [mt, open_source, lemmatizer] task: Lemmatization language: mt edition: Spark NLP 2.7.5 spark_version: 2.4 supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a dictionary-based lemmatizer that assigns all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_mt_2.7.5_2.4_1617376734828.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_mt_2.7.5_2.4_1617376734828.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | lemma| +-------+ | Il| | _| | _| | tal| | _| | _| | Leo| |Brincat| | _| | _| | _| | _| | _| | _| | _| | tal| | _| | MEPA| | _| | _| +-------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|mt| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) version 2.7. ## Benchmarking ```bash Precision=0.078, Recall=0.073, F1-score=0.075 ``` --- layout: model title: Named Entity Recognition - BERT Small (OntoNotes) author: John Snow Labs name: onto_small_bert_L4_512 date: 2020-12-05 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ner, open_source, en] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Onto is a Named Entity Recognition (or NER) model trained on OntoNotes 5.0. It can extract up to 18 entities such as people, places, organizations, money, time, date, etc. This model uses the pretrained `small_bert_L4_512` embeddings model from the `BertEmbeddings` annotator as an input. ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_small_bert_L4_512_en_2.7.0_2.4_1607199400149.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_small_bert_L4_512_en_2.7.0_2.4_1607199400149.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... ner_onto = NerDLModel.pretrained("onto_small_bert_L4_512", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val ner_onto = NerDLModel.pretrained("onto_small_bert_L4_512", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.onto.bert.small_l4_512').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PERSON | |October 28, 1955 |DATE | |American |NORP | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PERSON | |May 2014 |DATE | |the 1970s |DATE | |1980s |DATE | |Seattle |GPE | |Washington |GPE | |Gates |PERSON | |Paul Allen |PERSON | |1975 |DATE | |Albuquerque |GPE | |New Mexico |GPE | |Gates |PERSON | |January 2000 |DATE | |the late 1990s |DATE | |Gates |PERSON | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_small_bert_L4_512| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source The model is trained based on data from [OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) ## Benchmarking ```bash Micro-average: prec: 0.8697573, rec: 0.8567398, f1: 0.8631994 CoNLL Eval: processed 152728 tokens with 11257 phrases; found: 11219 phrases; correct: 9557. accuracy: 97.17%; 9557 11257 11219 precision: 85.19%; recall: 84.90%; FB1: 85.04 CARDINAL: 804 935 958 precision: 83.92%; recall: 85.99%; FB1: 84.94 958 DATE: 1412 1602 1726 precision: 81.81%; recall: 88.14%; FB1: 84.86 1726 EVENT: 20 63 46 precision: 43.48%; recall: 31.75%; FB1: 36.70 46 FAC: 78 135 122 precision: 63.93%; recall: 57.78%; FB1: 60.70 122 GPE: 2066 2240 2185 precision: 94.55%; recall: 92.23%; FB1: 93.38 2185 LANGUAGE: 10 22 11 precision: 90.91%; recall: 45.45%; FB1: 60.61 11 LAW: 12 40 18 precision: 66.67%; recall: 30.00%; FB1: 41.38 18 LOC: 114 179 168 precision: 67.86%; recall: 63.69%; FB1: 65.71 168 MONEY: 273 314 320 precision: 85.31%; recall: 86.94%; FB1: 86.12 320 NORP: 779 841 873 precision: 89.23%; recall: 92.63%; FB1: 90.90 873 ORDINAL: 174 195 226 precision: 76.99%; recall: 89.23%; FB1: 82.66 226 ORG: 1381 1795 1691 precision: 81.67%; recall: 76.94%; FB1: 79.23 1691 PERCENT: 311 349 349 precision: 89.11%; recall: 89.11%; FB1: 89.11 349 PERSON: 1827 1988 2046 precision: 89.30%; recall: 91.90%; FB1: 90.58 2046 PRODUCT: 32 76 51 precision: 62.75%; recall: 42.11%; FB1: 50.39 51 QUANTITY: 80 105 105 precision: 76.19%; recall: 76.19%; FB1: 76.19 105 TIME: 124 212 219 precision: 56.62%; recall: 58.49%; FB1: 57.54 219 WORK_OF_ART: 60 166 105 precision: 57.14%; recall: 36.14%; FB1: 44.28 105 ``` --- layout: model title: English image_classifier_vit_rust_image_classification_5 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_5 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_5` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_5_en_4.1.0_3.0_1660165828492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_5_en_4.1.0_3.0_1660165828492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_5", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_5", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Company to Ticker using Nasdaq author: John Snow Labs name: finel_nasdaq_data_ticker date: 2022-10-22 tags: [en, finance, companies, nasdaq, ticker, licensed] task: Entity Resolution language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Financial Entity Resolver model, trained to obtain the ticker from a Company Name, registered in NASDAQ. You can use this model after extracting a company name using any NER, and you will obtain its ticker. After this, you can use `finmapper_nasdaq_data_ticker` to augment and obtain more information about a company using NASDAQ datasource, including Official Company Name, Sector, Location, Currency, etc. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/ER_EDGAR_CRUNCHBASE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finel_nasdaq_data_ticker_en_1.0.0_3.0_1666473763228.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finel_nasdaq_data_ticker_en_1.0.0_3.0_1666473763228.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python test = ["FIDUS INVESTMENT corp","ASPECT DEVELOPMENT Inc","CFSB BANCORP","DALEEN TECHNOLOGIES","GLEASON Corporation"] testdf = pandas.DataFrame(test, columns=['text']) testsdf = spark.createDataFrame(testdf).toDF('text') documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("sentence") use = nlp.UniversalSentenceEncoder.pretrained("tfhub_use_lg", "en")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") use_er_model = finance.SentenceEntityResolverModel.pretrained('finel_nasdaq_data_ticker', 'en', 'finance/models')\ .setInputCols("embeddings")\ .setOutputCol('normalized')\ prediction_Model = nlp.Pipeline(stages=[documentAssembler, use, use_er_model]) test_pred = prediction_Model.transform(testsdf).cache() ```

## Results ```bash +----------------------+-------+ |text |result | +----------------------+-------+ |FIDUS INVESTMENT corp |[FDUS] | |ASPECT DEVELOPMENT Inc|[ASDV] | |CFSB BANCORP |[CFSB] | |DALEEN TECHNOLOGIES |[DALN1]| |GLEASON Corporation |[GLE1] | +----------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finel_nasdaq_data_ticker| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[normalized]| |Language:|en| |Size:|69.8 MB| |Case sensitive:|false| ## References NASDAQ Database --- layout: model title: German BERT Base Uncased Model author: John Snow Labs name: bert_base_german_uncased date: 2021-05-20 tags: [open_source, embeddings, german, de, bert] task: Embeddings language: de edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The source data for the model consists of a recent Wikipedia dump, EU Bookshop corpus, Open Subtitles, CommonCrawl, ParaCrawl and News Crawl. This results in a dataset with a size of 16GB and 2,350,234,427 tokens. The model is trained with an initial sequence length of 512 subwords and was performed for 1.5M steps. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_german_uncased_de_3.1.0_2.4_1621504361619.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_german_uncased_de_3.1.0_2.4_1621504361619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_german_uncased", "de") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_german_uncased", "de") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("de.embed.bert.uncased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_german_uncased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|de| |Case sensitive:|true| ## Data Source https://huggingface.co/dbmdz/bert-base-german-uncased ## Benchmarking ```bash For results on downstream tasks like NER or PoS tagging, please refer to [this repository](https://github.com/stefan-it/fine-tuned-berts-seq). ``` --- layout: model title: Fast Neural Machine Translation Model from English to Bemba (Zambia) author: John Snow Labs name: opus_mt_en_bem date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bem, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bem` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bem_xx_2.7.0_2.4_1609171023473.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bem_xx_2.7.0_2.4_1609171023473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bem| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Generic PHI for Deidentification (Arabic) author: John Snow Labs name: ner_deid_generic date: 2023-05-30 tags: [licensed, ner, clinical, deidentifiction, generic, arabic, ar] task: Named Entity Recognition language: ar edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Arabic) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 8 entities. This NER model is trained with a combination of custom datasets, and several data augmentation mechanisms. This model Word2Vec Arabic Clinical Embeddings. ## Predicted Entities `CONTACT`, `NAME`, `DATE`, `ID`, `SEX`, `LOCATION`, `PROFESSION`, `AGE` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_ar_4.4.2_3.0_1685443881012.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_ar_4.4.2_3.0_1685443881012.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+---------------------+ |chunk | ner_label | +-----------------+---------------------+ |30 مايو |DATE | |أحمد سليمان |NAME | |الرياض |LOCATION | |54321 |LOCATION | |المملكة العربية |LOCATION | |السعودية |LOCATION | |مستشفى الأمانة |LOCATION | |ريم الحمد |NAME | |أحمد |NAME | +---------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ar| |Size:|15.0 MB| ## References Custom John Snow Labs datasets Data augmentation techniques ## Benchmarking ```bash label tp fp fn total precision recall f1 CONTACT 146.0 0.0 6.0 152.0 1.0 0.9605 0.9799 NAME 685.0 25.0 25.0 710.0 0.9648 0.9648 0.9648 DATE 876.0 14.0 9.0 885.0 0.9843 0.9898 0.987 ID 28.0 9.0 2.0 30.0 0.7568 0.9333 0.8358 SEX 300.0 8.0 69.0 369.0 0.974 0.813 0.8863 LOCATION 689.0 48.0 38.0 727.0 0.9349 0.9477 0.9413 PROFESSION 303.0 20.0 32.0 335.0 0.9381 0.9045 0.921 AGE 608.0 7.0 9.0 617.0 0.9886 0.9854 0.987 macro - - - - - - 0.9378 micro - - - - - - 0.9572 ``` --- layout: model title: Clean documents pipeline for English author: John Snow Labs name: clean_stop date: 2021-03-24 tags: [open_source, english, clean_stop, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The clean_stop is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/clean_stop_en_3.0.0_3.0_1616544492033.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/clean_stop_en_3.0.0_3.0_1616544492033.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('clean_stop', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("clean_stop", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.clean.stop').predict(text) result_df ```

## Results ```bash | | document | sentence | token | cleanTokens | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:---------------------------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | ['Hello', 'John', 'Snow', 'Labs', '!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clean_stop| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: English XlmRoBertaForQuestionAnswering (from alon-albalak) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_large_xquad date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-large-xquad` is a English model originally trained by `alon-albalak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_xquad_en_4.0.0_3.0_1655996505419.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_xquad_en_4.0.0_3.0_1655996505419.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_large_xquad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_large_xquad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xquad.xlm_roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_large_xquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/alon-albalak/xlm-roberta-large-xquad - https://github.com/deepmind/xquad --- layout: model title: Fast Neural Machine Translation Model from English to Chinese author: John Snow Labs name: opus_mt_en_zh date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, zh, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `zh` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_zh_xx_2.7.0_2.4_1609168259647.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_zh_xx_2.7.0_2.4_1609168259647.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_zh| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_name_vsv_all_901529445 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-name_vsv_all-901529445` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Name`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1678783317887.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1678783317887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_name_vsv_all_901529445| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-name_vsv_all-901529445 --- layout: model title: English XlmRoBertaForQuestionAnswering (from deepset) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_squad2_distilled date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-squad2-distilled` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_squad2_distilled_en_4.0.0_3.0_1655991460437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_squad2_distilled_en_4.0.0_3.0_1655991460437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_squad2_distilled","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_squad2_distilled","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.distilled_base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_squad2_distilled| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|854.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/xlm-roberta-base-squad2-distilled - https://www.linkedin.com/company/deepset-ai/ - https://twitter.com/deepset_ai - http://www.deepset.ai/jobs - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/ - https://github.com/deepset-ai/FARM - https://deepset.ai/germanquad - https://deepset.ai - https://deepset.ai/german-bert - https://github.com/deepset-ai/haystack/discussions --- layout: model title: English DistilBertForQuestionAnswering model (from vkrishnamoorthy) author: John Snow Labs name: distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `vkrishnamoorthy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726595722.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726595722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_vkrishnamoorthy").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vkrishnamoorthy/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Detect Restaurant-related Terminology author: John Snow Labs name: nerdl_restaurant_100d_pipeline date: 2022-03-18 tags: [restaurant, ner, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [nerdl_restaurant_100d](https://nlp.johnsnowlabs.com/2021/12/31/nerdl_restaurant_100d_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_RESTAURANT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_RESTAURANT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_restaurant_100d_pipeline_en_3.4.1_3.0_1647610686318.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_restaurant_100d_pipeline_en_3.4.1_3.0_1647610686318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python restaurant_pipeline = PretrainedPipeline("nerdl_restaurant_100d_pipeline", lang = "en") restaurant_pipeline.annotate("Hong Kong’s favourite pasta bar also offers one of the most reasonably priced lunch sets in town! With locations spread out all over the territory Sha Tin – Pici’s formidable lunch menu reads like a highlight reel of the restaurant. Choose from starters like the burrata and arugula salad or freshly tossed tuna tartare, and reliable handmade pasta dishes like pappardelle. Finally, round out your effortless Italian meal with a tidy one-pot tiramisu, of course, an espresso to power you through the rest of the day.") ``` ```scala val restaurant_pipeline = new PretrainedPipeline("nerdl_restaurant_100d_pipeline", lang = "en") restaurant_pipeline.annotate("Hong Kong’s favourite pasta bar also offers one of the most reasonably priced lunch sets in town! With locations spread out all over the territory Sha Tin – Pici’s formidable lunch menu reads like a highlight reel of the restaurant. Choose from starters like the burrata and arugula salad or freshly tossed tuna tartare, and reliable handmade pasta dishes like pappardelle. Finally, round out your effortless Italian meal with a tidy one-pot tiramisu, of course, an espresso to power you through the rest of the day.") ```

## Results ```bash +---------------------------+---------------+ |chunk |ner_label | +---------------------------+---------------+ |Hong Kong’s |Restaurant_Name| |favourite |Rating | |pasta bar |Dish | |most reasonably |Price | |lunch |Hours | |in town! |Location | |Sha Tin – Pici’s |Restaurant_Name| |burrata |Dish | |arugula salad |Dish | |freshly tossed tuna tartare|Dish | |reliable |Price | |handmade pasta |Dish | |pappardelle |Dish | |effortless |Amenity | |Italian |Cuisine | |tidy one-pot |Amenity | |espresso |Dish | +---------------------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_restaurant_100d_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|166.7 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter - Finisher --- layout: model title: Social Determinants of Health author: John Snow Labs name: ner_sdoh date: 2023-06-13 tags: [clinical, en, social_determinants, ner, public_health, sdoh, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SDOH NER model is designed to detect and label social determinants of health (SDOH) entities within text data. Social determinants of health are crucial factors that influence individuals' health outcomes, encompassing various social, economic, and environmental element. The model has been trained using advanced machine learning techniques on a diverse range of text sources. The model can accurately recognize and classify a wide range of SDOH entities, including but not limited to factors such as socioeconomic status, education level, housing conditions, access to healthcare services, employment status, cultural and ethnic background, neighborhood characteristics, and environmental factors. The model's accuracy and precision have been carefully validated against expert-labeled data to ensure reliable and consistent results. Here are the labels of the SDOH NER model with their description: - `Access_To_Care`: Patient’s ability or barriers to access the care needed. "long distances, access to health care, rehab program, etc." - `Age`: All mention of ages including "Newborn, Infant, Child, Teenager, Teenage, Adult, etc." - `Alcohol`: Mentions of alcohol drinking habit. - `Chidhood_Event`: Childhood events mentioned by the patient. "childhood trauma, childhood abuse, etc." - `Communicable_Disease`: Include all the communicable diseases. "HIV, hepatitis, tuberculosis, sexually transmitted diseases, etc." - `Community_Safety`: safety of the neighborhood or places of study or work. "dangerous neighborhood, safe area, etc." - `Diet`: Information regarding the patient’s dietary habits. "vegetarian, vegan, healthy foods, low-calorie diet, etc." - `Disability`: Mentions related to disability - `Eating_Disorder`: This entity is used to extract eating disorders. "anorexia, bulimia, pica, etc." - `Education`:Patient’s educational background - `Employment`: Patient or provider occupational titles. - `Environmental_Condition`: Conditions of the environment where people live. "pollution, air quality, noisy environment, etc." - `Exercise`: Mentions of the exercise habits of a patient. "exercise, physical activity, play football, go to the gym, etc." - `Family_Member`: Nouns that refer to a family member. "mother, father, brother, sister, etc." - `Financial_Status`: Financial status refers to the state and condition of the person’s finances. "financial decline, debt, bankruptcy, etc." - `Food_Insecurity`: Food insecurity is defined as a lack of consistent access to enough food for every person in a household to live an active, healthy life. "food insecurity, scarcity of protein, lack of food, etc." - `Gender`: Gender-specific nouns and pronouns - `Geographic_Entity`: Geographical location refers to a specific physical point on Earth. - `Healthcare_Institution`: Health care institution means every place, institution, building or agency. "hospital, clinic, trauma centers, etc." - `Housing`: Conditions of the patient’s living spaces. "homeless, housing, small apartment, etc." - `Hyperlipidemia`: Terms that indicate hyperlipidemia and relevant subtypes. "hyperlipidemia, hypercholesterolemia, elevated cholesterol, etc." - `Hypertension`: Terms related to hypertension. "hypertension, high blood pressure, etc." - `Income`: Information regarding the patient’s income - `Insurance_Status`: Information regarding the patient’s insurance status. "uninsured, insured, Medicare, Medicaid, etc." - `Language`: A system of conventional spoken, manual (signed) or written symbols by means of which human beings express themselves. "English, Spanish-speaking, bilingual, etc. " - `Legal_Issues`: Issues that have legal implications. "legal issues, legal problems, detention , in prison, etc." - `Marital_Status`: Terms that indicate the person’s marital status. - `Mental_Health`: Include all the mental, neurodegenerative and neurodevelopmental diagnosis, disorders, conditions or syndromes mentioned. "depression, anxiety, bipolar disorder, psychosis, etc." - `Obesity`: Terms related to the patient being obese. "obesity, overweight, etc." - `Other_Disease`: Include all the diseases mentioned. "psoriasis, thromboembolism, etc." - `Other_SDoH_Keywords`: This label is used to annotated terms or sentences that provide information about social determinants of health that are not already extracted under any other entity label. "minimal activities of daily living, ack of government programs, etc." - `Population_Group`: The population group that a person belongs to, that does not fall under any other entities. "refugee, prison patient, etc." - `Quality_Of_Life`: Quality of life refers to how an individual feels about their current station in life. " lower quality of life, profoundly impact his quality of life, etc." - `Race_Ethnicity`: The race and ethnicity categories include racial, ethnic, and national origins. - `Sexual_Activity`: Mentions of patient’s sexual behaviors. "monogamous, sexual activity, inconsistent condom use, etc." - `Sexual_Orientation`: Terms that are related to sexual orientations. "gay, bisexual, heterosexual, etc." - `Smoking`: mentions of smoking habit. "smoking, cigarette, tobacco, etc." - `Social_Exclusion`: Absence or lack of rights or accessibility to services or goods that are expected of the majority of the population. "social exclusion, social isolation, gender discrimination, etc." - `Social_Support`: he presence of friends, family or other people to turn to for comfort or help. "social support, live with family, etc." - `Spiritual_Beliefs`: Spirituality is concerned with beliefs beyond self, usually related to the existence of a superior being. "spiritual beliefs, religious beliefs, strong believer, etc." - `Substance_Duration`: The duration associated with the health behaviors. "for 2 years, 3 months, etc" - `Substance_Frequency`: The frequency associated with the health behaviors. "five days a week, daily, weekly, monthly, etc" - `Substance_Quantity`: The quantity associated with the health behaviors. "2 packs , 40 ounces,ten to twelve, modarate, etc" - `Substance_Use`: Mentions of illegal recreational drugs use. Include also substances that can create dependency including here caffeine and tea. "overdose, cocaine, illicit substance intoxication, coffee etc." - `Transportation`: mentions of accessibility to transportation means. "car, bus, train, etc." - `Violence_Or_Abuse`: Episodes of abuse or violence experienced and reported by the patient. "domestic violence, sexual abuse, etc." ## Predicted Entities `Access_To_Care`, `Age`, `Alcohol`, `Chidhood_Event`, `Communicable_Disease`, `Community_Safety`, `Diet`, `Disability`, `Eating_Disorder`, `Education`, `Employment`, `Environmental_Condition`, `Exercise`, `Family_Member`, `Financial_Status`, `Food_Insecurity`, `Gender`, `Geographic_Entity`, `Healthcare_Institution`, `Housing`, `Hyperlipidemia`, `Hypertension`, `Income`, `Insurance_Status`, `Language`, `Legal_Issues`, `Marital_Status`, `Mental_Health`, `Obesity`, `Other_Disease`, `Other_SDoH_Keywords`, `Population_Group`, `Quality_Of_Life`, `Race_Ethnicity`, `Sexual_Activity`, `Sexual_Orientation`, `Smoking`, `Social_Exclusion`, `Social_Support`, `Spiritual_Beliefs`, `Substance_Duration`, `Substance_Frequency`, `Substance_Quantity`, `Substance_Use`, `Transportation`, `Violence_Or_Abuse` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_NER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_en_4.4.3_3.0_1686654976160.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_en_4.4.3_3.0_1686654976160.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = [["""Smith is 55 years old, living in New York, a divorced Mexcian American woman with financial problems. She speaks Spanish and Portuguese. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and cannot access health insurance or paid sick leave. She has a son, a student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reports having her catholic faith as a means of support as well. She has a long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI in April and was due to court this week."""]] data = spark.createDataFrame(sample_texts).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("""Smith is 55 years old, living in New York, a divorced Mexcian American woman with financial problems. She speaks Spanish and Portuguese. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and cannot access health insurance or paid sick leave. She has a son, a student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reports having her catholic faith as a means of support as well. She has a long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI in April and was due to court this week.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +------------------+-----+---+-------------------+ |chunk |begin|end|ner_label | +------------------+-----+---+-------------------+ |55 years old |9 |20 |Age | |New York |33 |40 |Geographic_Entity | |divorced |45 |52 |Marital_Status | |Mexcian American |54 |69 |Race_Ethnicity | |woman |71 |75 |Gender | |financial problems|82 |99 |Financial_Status | |She |102 |104|Gender | |Spanish |113 |119|Language | |Portuguese |125 |134|Language | |She |137 |139|Gender | |apartment |153 |161|Housing | |She |164 |166|Gender | |diabetes |193 |200|Other_Disease | |hospitalizations |268 |283|Other_SDoH_Keywords| |cleaning assistant|342 |359|Employment | |health insurance |379 |394|Insurance_Status | |She |416 |418|Gender | |son |426 |428|Family_Member | |student |433 |439|Education | |college |444 |450|Education | |depression |482 |491|Mental_Health | |She |494 |496|Gender | |she |507 |509|Gender | |rehab |517 |521|Access_To_Care | |her |542 |544|Gender | |catholic faith |546 |559|Spiritual_Beliefs | |support |575 |581|Social_Support | |She |593 |595|Gender | |etoh abuse |619 |628|Alcohol | |her |644 |646|Gender | |teens |648 |652|Age | |She |655 |657|Gender | |she |667 |669|Gender | |daily |682 |686|Substance_Frequency| |drinker |688 |694|Alcohol | |30 years |700 |707|Substance_Duration | |drinking |724 |731|Alcohol | |beer |733 |736|Alcohol | |daily |738 |742|Substance_Frequency| |She |745 |747|Gender | |smokes |749 |754|Smoking | |a pack |756 |761|Substance_Quantity | |cigarettes |766 |775|Smoking | |a day |777 |781|Substance_Frequency| |She |784 |786|Gender | |DUI |792 |794|Legal_Issues | +------------------+-----+---+-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.0 MB| |Dependencies:|embeddings_clinical| ## References Internal SDOH Project ## Benchmarking ```bash label precision recall f1-score support Access_To_Care 0.88 0.88 0.88 544 Age 0.93 0.95 0.94 466 Alcohol 0.96 0.98 0.97 263 Chidhood_Event 1.00 0.53 0.69 19 Communicable_Disease 0.96 1.00 0.98 51 Community_Safety 0.95 0.85 0.89 65 Diet 0.89 0.75 0.82 89 Disability 0.93 0.97 0.95 38 Eating_Disorder 0.95 0.91 0.93 44 Education 0.89 0.94 0.91 67 Employment 0.94 0.95 0.95 2087 Environmental_Condition 0.95 1.00 0.97 18 Exercise 0.90 0.86 0.88 65 Family_Member 0.98 0.98 0.98 2061 Financial_Status 0.73 0.77 0.75 116 Food_Insecurity 0.81 0.88 0.84 43 Gender 0.99 0.98 0.98 4858 Geographic_Entity 0.95 0.84 0.89 106 Healthcare_Institution 0.94 0.98 0.96 721 Housing 0.93 0.88 0.90 369 Hyperlipidemia 0.86 0.60 0.71 10 Hypertension 0.88 0.77 0.82 30 Income 0.82 0.75 0.79 61 Insurance_Status 0.89 0.82 0.85 66 Language 0.96 0.92 0.94 26 Legal_Issues 0.89 0.79 0.84 63 Marital_Status 0.96 1.00 0.98 90 Mental_Health 0.87 0.85 0.86 491 Obesity 0.85 1.00 0.92 11 Other_Disease 0.90 0.89 0.90 609 Other_SDoH_Keywords 0.81 0.85 0.83 258 Population_Group 1.00 0.86 0.92 14 Quality_Of_Life 0.61 0.86 0.72 44 Race_Ethnicity 0.97 0.90 0.94 41 Sexual_Activity 0.78 0.84 0.81 45 Sexual_Orientation 0.80 1.00 0.89 12 Smoking 0.99 0.97 0.98 76 Social_Exclusion 0.88 0.88 0.88 25 Social_Support 0.87 0.95 0.91 676 Spiritual_Beliefs 0.83 0.83 0.83 53 Substance_Duration 0.78 0.69 0.73 52 Substance_Frequency 0.52 0.83 0.64 46 Substance_Quantity 0.82 0.88 0.85 51 Substance_Use 0.91 0.97 0.94 207 Transportation 0.85 0.83 0.84 42 Violence_Or_Abuse 0.87 0.70 0.77 112 micro-avg 0.94 0.94 0.94 15301 macro-avg 0.88 0.87 0.87 15301 weighted-avg 0.94 0.94 0.94 15301 ``` --- layout: model title: English BertForQuestionAnswering model (from FardinSaboori) author: John Snow Labs name: bert_qa_FardinSaboori_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `FardinSaboori`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_FardinSaboori_bert_finetuned_squad_en_4.0.0_3.0_1654535437430.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_FardinSaboori_bert_finetuned_squad_en_4.0.0_3.0_1654535437430.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_FardinSaboori_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_FardinSaboori_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_FardinSaboori").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_FardinSaboori_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/FardinSaboori/bert-finetuned-squad --- layout: model title: Bangla Bert Embeddings (from Kowsher) author: John Snow Labs name: bert_embeddings_bangla_bert date: 2022-04-11 tags: [bert, embeddings, bn, open_source] task: Embeddings language: bn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bangla-bert` is a Bangla model orginally trained by `Kowsher`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_bert_bn_3.4.2_3.0_1649673360956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_bert_bn_3.4.2_3.0_1649673360956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bangla_bert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|615.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/Kowsher/bangla-bert - https://github.com/Kowsher/bert-base-bangla - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://www.kaggle.com/gakowsher/bangla-language-model-dataset - https://ssrn.com/abstract= - http://kowsher.org/ --- layout: model title: Translate Central Bikol to English Pipeline author: John Snow Labs name: translate_bcl_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bcl, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bcl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bcl_en_xx_2.7.0_2.4_1609688692603.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bcl_en_xx_2.7.0_2.4_1609688692603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bcl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bcl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bcl.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bcl_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Seychellois Creole Pipeline author: John Snow Labs name: translate_en_crs date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, crs, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `crs` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_crs_xx_2.7.0_2.4_1609691811394.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_crs_xx_2.7.0_2.4_1609691811394.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_crs", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_crs", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.crs').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_crs| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from Wiam) author: John Snow Labs name: distilbert_qa_Wiam_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Wiam`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Wiam_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724868151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Wiam_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724868151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Wiam_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Wiam_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Wiam").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Wiam_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Wiam/distilbert-base-uncased-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering (from deepset) author: John Snow Labs name: roberta_qa_roberta_base_squad2_distilled date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-distilled` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_distilled_en_4.0.0_3.0_1655735282920.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_distilled_en_4.0.0_3.0_1655735282920.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_squad2_distilled","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_squad2_distilled","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.distilled_base.by_deepset").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_squad2_distilled| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/roberta-base-squad2-distilled - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/FARM - http://www.deepset.ai/jobs - https://twitter.com/deepset_ai - https://github.com/deepset-ai/haystack/discussions - https://github.com/deepset-ai/haystack/ - https://deepset.ai - https://deepset.ai/germanquad - https://deepset.ai/german-bert --- layout: model title: Mapping Entities (Clinical Drugs) with Corresponding UMLS CUI Codes author: John Snow Labs name: umls_clinical_drugs_mapper date: 2022-07-06 tags: [umls, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps entities (Clinical Drugs) with their corresponding UMLS CUI codes. ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_clinical_drugs_mapper_en_4.0.0_3.0_1657124255341.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_clinical_drugs_mapper_en_4.0.0_3.0_1657124255341.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("clinical_ner") ner_model_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "clinical_ner"])\ .setOutputCol("ner_chunk") chunkerMapper = ChunkMapperModel.pretrained("umls_clinical_drugs_mapper", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["umls_code"])\ .setLowerCase(True) mapper_pipeline = Pipeline().setStages([ document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunkerMapper]) sample_text="""She was immediately given hydrogen peroxide 30 mg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg.""" test_data = spark.createDataFrame([[sample_text]]).toDF("text") result = mapper_pipeline.fit(test_data).transform(test_data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel .pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("clinical_ner") val ner_model_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "clinical_ner")) .setOutputCol("ner_chunk") val chunkerMapper = ChunkMapperModel .pretrained("umls_clinical_drugs_mapper", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("mappings") .setRels(Array("umls_code")) val mapper_pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunkerMapper)) val test_data = Seq("She was immediately given hydrogen peroxide 30 mg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg.").toDF("text") val result = pipeline.fit(test_data).transform(test_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.umls_clinical_drugs_mapper").predict("""She was immediately given hydrogen peroxide 30 mg, and has been advised Neosporin Cream for 5 days. She has a history of taking magnesium hydroxide 100mg/1ml and metformin 1000 mg.""") ```

## Results ```bash +-------------------+---------+ |ner_chunk |umls_code| +-------------------+---------+ |hydrogen peroxide |C0020281 | |Neosporin Cream |C0132149 | |magnesium hydroxide|C0024476 | |metformin |C0025598 | +-------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_clinical_drugs_mapper| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|23.3 MB| ## References 2022AA UMLS dataset’s Clinical Drug category. https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: Detect Clinical Entities (ner_eu_clinical_case - fr) author: John Snow Labs name: ner_eu_clinical_case date: 2023-02-01 tags: [fr, clinical, licensed, ner] task: Named Entity Recognition language: fr edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition (NER) deep learning model for extracting clinical entities from French texts. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nichols, Named Entity Recognition with Bidirectional LSTM-CNN. The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Predicted Entities `clinical_event`, `bodypart`, `clinical_condition`, `units_measurements`, `patient`, `date_time` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_fr_4.2.8_3.0_1675293960896.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_fr_4.2.8_3.0_1675293960896.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","fr")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_eu_clinical_case', "fr", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""Un garçon de 3 ans atteint d'un trouble autistique à l'hôpital du service pédiatrique A de l'hôpital universitaire. Il n'a pas d'antécédents familiaux de troubles ou de maladies du spectre autistique. Le garçon a été diagnostiqué avec un trouble de communication sévère, avec des difficultés d'interaction sociale et un traitement sensoriel retardé. Les tests sanguins étaient normaux (thyréostimuline (TSH), hémoglobine, volume globulaire moyen (MCV) et ferritine). L'endoscopie haute a également montré une tumeur sous-muqueuse provoquant une obstruction subtotale de la sortie gastrique. Devant la suspicion d'une tumeur stromale gastro-intestinale, une gastrectomie distale a été réalisée. L'examen histopathologique a révélé une prolifération de cellules fusiformes dans la couche sous-muqueuse."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","fr") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_eu_clinical_case", "fr", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""Un garçon de 3 ans atteint d'un trouble autistique à l'hôpital du service pédiatrique A de l'hôpital universitaire. Il n'a pas d'antécédents familiaux de troubles ou de maladies du spectre autistique. Le garçon a été diagnostiqué avec un trouble de communication sévère, avec des difficultés d'interaction sociale et un traitement sensoriel retardé. Les tests sanguins étaient normaux (thyréostimuline (TSH), hémoglobine, volume globulaire moyen (MCV) et ferritine). L'endoscopie haute a également montré une tumeur sous-muqueuse provoquant une obstruction subtotale de la sortie gastrique. Devant la suspicion d'une tumeur stromale gastro-intestinale, une gastrectomie distale a été réalisée. L'examen histopathologique a révélé une prolifération de cellules fusiformes dans la couche sous-muqueuse.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-----------------------------------------------------+------------------+ |chunk |ner_label | +-----------------------------------------------------+------------------+ |Un garçon de 3 ans |patient | |trouble autistique à l'hôpital du service pédiatrique|clinical_condition| |l'hôpital |clinical_event | |Il n'a |patient | |d'antécédents |clinical_event | |troubles |clinical_condition| |maladies |clinical_condition| |du spectre autistique |bodypart | |Le garçon |patient | |diagnostiqué |clinical_event | |trouble |clinical_condition| |difficultés |clinical_event | |traitement |clinical_event | |tests |clinical_event | |normaux |units_measurements| |thyréostimuline |clinical_event | |TSH |clinical_event | |ferritine |clinical_event | |L'endoscopie |clinical_event | |montré |clinical_event | |tumeur sous-muqueuse |clinical_condition| |provoquant |clinical_event | |obstruction |clinical_condition| |la sortie gastrique |bodypart | |suspicion |clinical_event | |tumeur stromale gastro-intestinale |clinical_condition| |gastrectomie |clinical_event | |L'examen |clinical_event | |révélé |clinical_event | |prolifération |clinical_event | |cellules fusiformes |bodypart | |la couche sous-muqueuse |bodypart | +-----------------------------------------------------+------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_case| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|fr| |Size:|895.0 KB| ## References The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Benchmarking ```bash label tp fp fn total precision recall f1 date_time 49.0 14.0 70.0 104.0 0.7778 0.7000 0.7368 units_measurements 92.0 19.0 6.0 48.0 0.8288 0.9388 0.8804 clinical_condition 178.0 74.0 73.0 120.0 0.7063 0.7092 0.7078 patient 114.0 6.0 15.0 87.0 0.9500 0.8837 0.9157 clinical_event 265.0 81.0 71.0 478.0 0.7659 0.7887 0.7771 bodypart 243.0 34.0 64.0 166.0 0.8773 0.7915 0.8322 macro - - - - - - 0.8083 micro - - - - - - 0.7978 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from 21iridescent) author: John Snow Labs name: distilbert_qa_21iridescent_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `21iridescent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_21iridescent_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724023073.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_21iridescent_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724023073.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_21iridescent_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_21iridescent_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_21iridescent").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_21iridescent_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/21iridescent/distilbert-base-uncased-finetuned-squad --- layout: model title: English AlbertForQuestionAnswering model (from AyushPJ) author: John Snow Labs name: albert_qa_ai_club_inductions_21_nlp date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ai-club-inductions-21-nlp-ALBERT` is a English model originally trained by `AyushPJ`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_ai_club_inductions_21_nlp_en_4.0.0_3.0_1656063682959.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_ai_club_inductions_21_nlp_en_4.0.0_3.0_1656063682959.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_ai_club_inductions_21_nlp","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_ai_club_inductions_21_nlp","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.albert.by_AyushPJ").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_ai_club_inductions_21_nlp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AyushPJ/ai-club-inductions-21-nlp-ALBERT --- layout: model title: Spanish RobertaForQuestionAnswering (from mrm8488) author: John Snow Labs name: roberta_qa_RuPERTa_base_finetuned_squadv1 date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RuPERTa-base-finetuned-squadv1` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_RuPERTa_base_finetuned_squadv1_es_4.0.0_3.0_1655727321165.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_RuPERTa_base_finetuned_squadv1_es_4.0.0_3.0_1655727321165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_RuPERTa_base_finetuned_squadv1","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_RuPERTa_base_finetuned_squadv1","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.ruperta.base.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_RuPERTa_base_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|470.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-squadv1 --- layout: model title: Word2Vec Embeddings in Spanish (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_es_3.4.1_3.0_1647459363492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_es_3.4.1_3.0_1647459363492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = BertSentenceEmbeddings\ .pretrained('labse', 'xx') \ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") sentimentClassifier = ClassifierDLModel.pretrained("classifierdl_bert_sentiment", "fr") \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") fr_sentiment_pipeline = Pipeline(stages=[document, embeddings, sentimentClassifier]) light_pipeline = LightPipeline(fr_sentiment_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result1 = light_pipeline.annotate("Mignolet vraiment dommage de ne jamais le voir comme titulaire") result2 = light_pipeline.annotate("Je me sens bien, je suis heureux d'être de retour.") print(result1["class"], result2["class"], sep = "\n") ``` ```scala val document = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val embeddings = BertSentenceEmbeddings .pretrained("labse", "xx") .setInputCols(Array("document")) .setOutputCol("sentence_embeddings") val sentimentClassifier = ClassifierDLModel.pretrained("classifierdl_bert_sentiment", "fr") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val fr_sentiment_pipeline = new Pipeline().setStages(Array(document, embeddings, sentimentClassifier)) val light_pipeline = LightPipeline(fr_sentiment_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) val result1 = light_pipeline.annotate("Mignolet vraiment dommage de ne jamais le voir comme titulaire") val result2 = light_pipeline.annotate("Je me sens bien, je suis heureux d'être de retour.") ``` {:.nlu-block} ```python import nlu nlu.load("fr.classify.sentiment.bert").predict("""Mignolet vraiment dommage de ne jamais le voir comme titulaire""") ```

## Results ```bash ['NEGATIVE'] ['POSITIVE'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_sentiment| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|fr| ## Data Source https://github.com/charlesmalafosse/open-dataset-for-sentiment-analysis/ ## Benchmarking ```bash precision recall f1-score support NEGATIVE 0.82 0.72 0.77 378 POSITIVE 0.92 0.95 0.94 1240 accuracy 0.90 1618 macro avg 0.87 0.84 0.85 1618 weighted avg 0.90 0.90 0.90 1618 ``` --- layout: model title: Legal Confidential Clause Binary Classifier author: John Snow Labs name: legclf_confidential_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `confidential` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `confidential` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_confidential_clause_en_1.0.0_3.2_1660122270121.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_confidential_clause_en_1.0.0_3.2_1660122270121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_confidential_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[confidential]| |[other]| |[other]| |[confidential]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_confidential_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support confidential 0.90 0.90 0.90 41 other 0.97 0.97 0.97 127 accuracy - - 0.95 168 macro-avg 0.94 0.94 0.94 168 weighted-avg 0.95 0.95 0.95 168 ``` --- layout: model title: Stop Words Cleaner for Sesotho author: John Snow Labs name: stopwords_st date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: st edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, st] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_st_st_2.5.4_2.4_1594742438831.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_st_st_2.5.4_2.4_1594742438831.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_st", "st") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Ntle le ho ba morena oa leboea, John Snow ke ngaka ea Lenyesemane ebile ke moetapele nts'etsopele ea anesthesia le bohloeki ba bongaka.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_st", "st") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Ntle le ho ba morena oa leboea, John Snow ke ngaka ea Lenyesemane ebile ke moetapele nts'etsopele ea anesthesia le bohloeki ba bongaka.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Ntle le ho ba morena oa leboea, John Snow ke ngaka ea Lenyesemane ebile ke moetapele nts'etsopele ea anesthesia le bohloeki ba bongaka."""] stopword_df = nlu.load('st.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=3, result='Ntle', metadata={'sentence': '0'}), Row(annotatorType='token', begin=14, end=19, result='morena', metadata={'sentence': '0'}), Row(annotatorType='token', begin=24, end=29, result='leboea', metadata={'sentence': '0'}), Row(annotatorType='token', begin=30, end=30, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=32, end=35, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_st| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|st| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English Bert Embeddings Cased model (from Tristan) author: John Snow Labs name: bert_embeddings_olm_base_uncased_oct_2022 date: 2023-02-21 tags: [open_source, bert, bert_embeddings, bertformaskedlm, en, tensorflow] task: Embeddings language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `olm-bert-base-uncased-oct-2022` is a English model originally trained by `Tristan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_olm_base_uncased_oct_2022_en_4.3.0_3.0_1676999449577.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_olm_base_uncased_oct_2022_en_4.3.0_3.0_1676999449577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_olm_base_uncased_oct_2022","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_olm_base_uncased_oct_2022","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_olm_base_uncased_oct_2022| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|467.5 MB| |Case sensitive:|true| ## References https://huggingface.co/Tristan/olm-bert-base-uncased-oct-2022 --- layout: model title: Detect PHI for Deidentification in Romanian (BERT) author: John Snow Labs name: ner_deid_subentity_bert date: 2022-06-27 tags: [deidentification, bert, phi, ner, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It is trained with `bert_base_cased` embeddings and can detect 17 entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CITY`, `COUNTRY`, `DATE`, `DOCTOR`, `EMAIL`, `FAX`, `HOSPITAL`, `IDNUM`, `LOCATION-OTHER`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `ZIP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_bert_ro_4.0.0_3.0_1656311815383.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_bert_ro_4.0.0_3.0_1656311815383.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|HOSPITAL | |Drumul Oprea Nr |STREET | |Vaslui |CITY | |737405 |ZIP | |+40(235)413773 |PHONE | |25 May 2022 |DATE | |BUREAN MARIA |PATIENT | |77 |AGE | |Agota Evelyn Tımar |DOCTOR | |2450502264401 |IDNUM | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_bert| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.5 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.98 0.95 0.96 1186 CITY 0.94 0.87 0.90 299 COUNTRY 0.90 0.73 0.81 108 DATE 0.98 0.95 0.96 4518 DOCTOR 0.91 0.94 0.93 1979 EMAIL 1.00 0.62 0.77 8 FAX 0.98 0.95 0.96 56 HOSPITAL 0.92 0.85 0.88 881 IDNUM 0.98 0.99 0.98 235 LOCATION-OTHER 1.00 0.85 0.92 13 MEDICALRECORD 0.99 1.00 1.00 444 ORGANIZATION 0.86 0.76 0.81 75 PATIENT 0.91 0.87 0.89 937 PHONE 0.96 0.98 0.97 302 PROFESSION 0.85 0.82 0.83 161 STREET 0.96 0.94 0.95 173 ZIP 0.99 0.98 0.99 138 micro-avg 0.95 0.93 0.94 11513 macro-avg 0.95 0.89 0.91 11513 weighted-avg 0.95 0.93 0.94 11513 ``` --- layout: model title: English BertForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: bert_qa_bert_base_cased_chaii date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-chaii` is a English model orginally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_cased_chaii_en_4.0.0_3.0_1654179712101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_cased_chaii_en_4.0.0_3.0_1654179712101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_cased_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_cased_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.bert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_cased_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/bert-base-cased-chaii --- layout: model title: Legal Arguments Mining in Court Decisions author: John Snow Labs name: legclf_argument_mining date: 2023-03-26 tags: [en, classification, licensed, legal, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model which classifies arguments in legal discourse. These are the following classes: `subsumption`, `definition`, `conclusion`, `other`. ## Predicted Entities `subsumption`, `definition`, `conclusion`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_argument_mining_en_1.0.0_3.0_1679829561976.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_argument_mining_en_1.0.0_3.0_1679829561976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base", "en")\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) embeddingsSentence = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"])\ .setOutputCol("sentence_embeddings")\ .setPoolingStrategy("AVERAGE") docClassifier = legal.ClassifierDLModel.pretrained("legclf_argument_mining","en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, embeddingsSentence, docClassifier ]) df = spark.createDataFrame([["There is therefore no doubt – and the Government do not contest – that the measures concerned in the present case ( the children 's continued placement in foster homes and the restrictions imposed on contact between the applicants and their children ) amounts to an “ interference ” with the applicants ' rights to respect for their family life ."]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) result.select("text", "category.result").show() ```

## Results ```bash +--------------------+-------------+ | text| result| +--------------------+-------------+ |There is therefor...|[subsumption]| +--------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_argument_mining| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/MeilingShi/legal_argument_mining) ## Benchmarking ```bash label precision recall f1-score support conclusion 0.93 0.79 0.85 52 definition 0.87 0.81 0.84 58 other 0.88 0.88 0.88 57 subsumption 0.64 0.79 0.71 52 accuracy - - 0.82 219 macro-avg 0.83 0.82 0.82 219 weighted-avg 0.83 0.82 0.82 219 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_hfl_chinese_roberta_wwm_ext date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-roberta-wwm-ext` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_hfl_chinese_roberta_wwm_ext_zh_4.2.4_3.0_1670021322707.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_hfl_chinese_roberta_wwm_ext_zh_4.2.4_3.0_1670021322707.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_hfl_chinese_roberta_wwm_ext","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_hfl_chinese_roberta_wwm_ext","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_hfl_chinese_roberta_wwm_ext| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-roberta-wwm-ext - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English BertForQuestionAnswering model (from peterhsu) author: John Snow Labs name: bert_qa_peterhsu_bert_finetuned_squad_accelerate date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model orginally trained by `peterhsu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_peterhsu_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535878232.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_peterhsu_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535878232.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_peterhsu_bert_finetuned_squad_accelerate","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_peterhsu_bert_finetuned_squad_accelerate","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.accelerate.by_peterhsu").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_peterhsu_bert_finetuned_squad_accelerate| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/peterhsu/bert-finetuned-squad-accelerate --- layout: model title: Fast Neural Machine Translation Model from Estonian to English author: John Snow Labs name: opus_mt_et_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, et, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `et` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_et_en_xx_2.7.0_2.4_1609170283601.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_et_en_xx_2.7.0_2.4_1609170283601.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_et_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_timit_moaiz_exp2 TFWav2Vec2ForCTC from moaiz237 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_moaiz_exp2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_moaiz_exp2` is a English model originally trained by moaiz237. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_moaiz_exp2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_moaiz_exp2_en_4.2.0_3.0_1664037629984.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_moaiz_exp2_en_4.2.0_3.0_1664037629984.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_moaiz_exp2', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_moaiz_exp2", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_moaiz_exp2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-28 tags: [ga, licensed, ner, legal, mapa] task: Named Entity Recognition language: ga edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Irish` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_ga_1.0.0_3.0_1682670223837.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_ga_1.0.0_3.0_1682670223837.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_irish_legal","gle")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "ga", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Dhiúltaigh Tribunale di Teramo ( An Chúirt Dúiche, Teramo ) an t-iarratas a rinne Bn.Grigorescu, ar bhonn teagmhasach, chun aitheantas a thabhairt san Iodáil do bhreithiúnas colscartha Tribunalul București ( An Chúirt Réigiúnach, Búcairist ) an 3 Nollaig 2012, de bhun Rialachán Uimh."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |Teramo |ADDRESS | |Bn.Grigorescu |PERSON | |Búcairist |ADDRESS | |3 Nollaig 2012|DATE | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ga| |Size:|16.3 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.82 0.74 0.78 19 AMOUNT 1.00 1.00 1.00 7 DATE 0.91 0.92 0.91 75 ORGANISATION 0.65 0.67 0.66 48 PERSON 0.71 0.82 0.76 56 micro-avg 0.79 0.82 0.80 205 macro-avg 0.82 0.83 0.82 205 weighted-avg 0.79 0.82 0.80 205 ``` --- layout: model title: English asr_distil_wav2vec2 TFWav2Vec2ForCTC from OthmaneJ author: John Snow Labs name: asr_distil_wav2vec2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_distil_wav2vec2` is a English model originally trained by OthmaneJ. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_distil_wav2vec2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_distil_wav2vec2_en_4.2.0_3.0_1664020967214.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_distil_wav2vec2_en_4.2.0_3.0_1664020967214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_distil_wav2vec2", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_distil_wav2vec2", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_distil_wav2vec2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|188.9 MB| --- layout: model title: Legal BERT Base Uncased Embedding author: John Snow Labs name: bert_base_uncased_legal date: 2021-09-07 tags: [english, legal, open_source, bert_embeddings, uncased, en] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEGAL-BERT is a family of BERT models for the legal domain, intended to assist legal NLP research, computational law, and legal technology applications. To pre-train the different variations of LEGAL-BERT, we collected 12 GB of diverse English legal text from several fields (e.g., legislation, court cases, contracts) scraped from publicly available resources. Sub-domains variants (CONTRACTS-, EURLEX-, ECHR-) and/or general LEGAL-BERT perform better than using BERT out of the box for domain-specific tasks. A light-weight model (33% the size of BERT-BASE) pre-trained from scratch on legal data with competitive perfomance is also available. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_uncased_legal_en_3.2.2_3.0_1630999701913.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_uncased_legal_en_3.2.2_3.0_1630999701913.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_uncased_legal", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_uncased_legal", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.bert.base_uncased_legal").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_uncased_legal| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/nlpaueb/legal-bert-base-uncased --- layout: model title: English asr_wav2vec2_base_timit_demo_colab240 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab240 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab240` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab240_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab240_en_4.2.0_3.0_1664023921297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab240_en_4.2.0_3.0_1664023921297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab240", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab240", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab240| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Turkish Named Entity Recognition (from akdeniz27) author: John Snow Labs name: bert_ner_bert_base_turkish_cased_ner date: 2022-05-09 tags: [bert, ner, token_classification, tr, open_source] task: Named Entity Recognition language: tr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-turkish-cased-ner` is a Turkish model orginally trained by `akdeniz27`. ## Predicted Entities `LOC`, `PER`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_turkish_cased_ner_tr_3.4.2_3.0_1652099217326.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_turkish_cased_ner_tr_3.4.2_3.0_1652099217326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_turkish_cased_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|tr| |Size:|412.9 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/akdeniz27/bert-base-turkish-cased-ner - https://github.com/stefan-it/turkish-bert/files/4558187/nerdata.txt - https://ieeexplore.ieee.org/document/7495744 --- layout: model title: Sentence Entity Resolver for ICD10-CM (Augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_augmented date: 2022-01-18 tags: [icd10cm, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it has been augmented with synonyms for making it more accurate. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_2.4_1642532480732.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_2.4_1642532480732.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk| entity|icd10cm_code| resolutions| all_codes| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O2441|gestational diabetes mellitus:::postpartum gestational diabetes mel...| O2441:::O2443:::Z8632:::Z875:::O2431:::O2411:::O244:::O241:::O2481| |subsequent type two diabetes mellitus|PROBLEM| O2411|pre-existing type 2 diabetes mellitus:::disorder associated with ty...|O2411:::E118:::E11:::E139:::E119:::E113:::E1144:::Z863:::Z8639:::E1...| | T2DM|PROBLEM| E11|type 2 diabetes mellitus:::disorder associated with type 2 diabetes...|E11:::E118:::E119:::O2411:::E109:::E139:::E113:::E8881:::Z833:::D64...| | HTG-induced pancreatitis|PROBLEM| K8520|alcohol-induced pancreatitis:::drug-induced acute pancreatitis:::he...|K8520:::K853:::K8590:::F102:::K852:::K859:::K8580:::K8591:::K858:::...| | acute hepatitis|PROBLEM| K720|acute hepatitis:::acute hepatitis a:::acute infectious hepatitis:::...|K720:::B15:::B179:::B172:::Z0389:::B159:::B150:::B16:::K752:::K712:...| | obesity|PROBLEM| E669|obesity:::abdominal obesity:::obese:::central obesity:::overweight ...|E669:::E668:::Z6841:::Q130:::E66:::E6601:::Z8639:::E349:::H3550:::Z...| | a body mass index|PROBLEM| Z6841|finding of body mass index:::observation of body mass index:::mass ...|Z6841:::E669:::R229:::Z681:::R223:::R221:::Z68:::R222:::R220:::R418...| | polyuria|PROBLEM| R35|polyuria:::nocturnal polyuria:::polyuric state:::polyuric state (di...|R35:::R3581:::R358:::E232:::R31:::R350:::R8299:::N401:::E723:::O048...| | polydipsia|PROBLEM| R631|polydipsia:::psychogenic polydipsia:::primary polydipsia:::psychoge...|R631:::F6389:::E232:::F639:::O40:::G475:::M7989:::R632:::R061:::H53...| | poor appetite|PROBLEM| R630|poor appetite:::poor feeding:::bad taste in mouth:::unpleasant tast...|R630:::P929:::R438:::R432:::E86:::R196:::F520:::Z724:::R0689:::Z768...| | vomiting|PROBLEM| R111|vomiting:::intermittent vomiting:::vomiting symptoms:::periodic vom...| R111:::R11:::R1110:::G43A1:::P921:::P9209:::G43A:::R1113:::R110| | a respiratory tract infection|PROBLEM| J988|respiratory tract infection:::upper respiratory tract infection:::b...|J988:::J069:::A499:::J22:::J209:::Z593:::T17:::J0410:::Z1383:::J189...| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|1.4 GB| |Case sensitive:|false| |Dependencies:|embeddings_clinical| ## Data Source Trained on ICD10CM 2022 Codes dataset: https://www.cdc.gov/nchs/icd/icd10cm.htm --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265911` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911_en_4.0.0_3.0_1655985889976.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911_en_4.0.0_3.0_1655985889976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265911").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265911| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265911 --- layout: model title: Legal Return Of Confidential Information Clause Binary Classifier author: John Snow Labs name: legclf_return_of_conf_info_clause date: 2023-02-13 tags: [en, legal, classification, return, confidential, information, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `return_of_conf_info` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `return_of_conf_info`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_return_of_conf_info_clause_en_1.0.0_3.0_1676304098427.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_return_of_conf_info_clause_en_1.0.0_3.0_1676304098427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_return_of_conf_info_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[return_of_conf_info]| |[other]| |[other]| |[return_of_conf_info]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_return_of_conf_info_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 8 return_of_conf_info 1.00 1.00 1.00 15 accuracy - - 1.00 23 macro-avg 1.00 1.00 1.00 23 weighted-avg 1.00 1.00 1.00 23 ``` --- layout: model title: Legal Limited Liability Company Agreement Document Binary Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_limited_liability_company_agreement_bert date: 2022-12-18 tags: [en, legal, classification, licensed, document, bert, limited, liability, company, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_limited_liability_company_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `limited-liability-company-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `limited-liability-company-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_limited_liability_company_agreement_bert_en_1.0.0_3.0_1671393854665.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_limited_liability_company_agreement_bert_en_1.0.0_3.0_1671393854665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_limited_liability_company_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[limited-liability-company-agreement]| |[other]| |[other]| |[limited-liability-company-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_limited_liability_company_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support limited-liability-company-agreement 0.98 0.98 0.98 121 other 0.99 0.99 0.99 204 accuracy - - 0.98 325 macro-avg 0.98 0.98 0.98 325 weighted-avg 0.98 0.98 0.98 325 ``` --- layout: model title: NER Pipeline for 10 High Resourced Languages author: John Snow Labs name: xlm_roberta_large_token_classifier_hrl_pipeline date: 2022-06-27 tags: [arabic, german, english, spanish, french, italian, latvian, dutch, portuguese, chinese, xlm, roberta, ner, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_large_token_classifier_hrl](https://nlp.johnsnowlabs.com/2021/12/26/xlm_roberta_large_token_classifier_hrl_xx.html) model. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_HRL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Ner_HRL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_hrl_pipeline_xx_4.0.0_3.0_1656371823877.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_hrl_pipeline_xx_4.0.0_3.0_1656371823877.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlm_roberta_large_token_classifier_hrl_pipeline", lang = "xx") pipeline.annotate("يمكنكم مشاهدة أمير منطقة الرياض الأمير فيصل بن بندر بن عبد العزيز في كل مناسبة وافتتاح تتعلق بمشاريع التعليم والصحة وخدمة الطرق والمشاريع الثقافية في منطقة الرياض.") ``` ```scala val pipeline = new PretrainedPipeline("xlm_roberta_large_token_classifier_hrl_pipeline", lang = "xx") pipeline.annotate("يمكنكم مشاهدة أمير منطقة الرياض الأمير فيصل بن بندر بن عبد العزيز في كل مناسبة وافتتاح تتعلق بمشاريع التعليم والصحة وخدمة الطرق والمشاريع الثقافية في منطقة الرياض.") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |الرياض |LOC | |فيصل بن بندر بن عبد العزيز |PER | |الرياض |LOC | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_large_token_classifier_hrl_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|xx| |Size:|1.8 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - XlmRoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: Explain Document Pipeline for Finnish author: John Snow Labs name: explain_document_sm date: 2021-03-22 tags: [open_source, finnish, explain_document_sm, pipeline, fi] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: fi edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_sm_fi_3.0.0_3.0_1616429037499.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_sm_fi_3.0.0_3.0_1616429037499.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_sm', lang = 'fi') annotations = pipeline.fullAnnotate(""Hei John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_sm", lang = "fi") val result = pipeline.fullAnnotate("Hei John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei John Snow Labs! ""] result_df = nlu.load('fi.explain').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-------------------------|:------------------------|:---------------------------------|:---------------------------------|:------------------------------------|:-----------------------------|:---------------------------------|:--------------------| | 0 | ['Hei John Snow Labs! '] | ['Hei John Snow Labs!'] | ['Hei', 'John', 'Snow', 'Labs!'] | ['hei', 'John', 'Snow', 'Labs!'] | ['INTJ', 'PROPN', 'PROPN', 'PROPN'] | [[-0.394499987363815,.,...]] | ['O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm_augmented date: 2022-01-03 tags: [rxnorm, licensed, en, clinical, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It trained on the augmented version of the dataset which is used in previous RxNorm resolver models. Additionally, this model returns concept classes of the drugs in `all_k_aux_labels` column. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_en_3.3.1_2.4_1641241820334.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_en_3.3.1_2.4_1641241820334.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_rxnorm_augmented``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_posology``` as NER model. ```DRUG``` set in ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli", "en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxnorm_augmented", "en", "clinical/models")\ .setInputCols("sbert_embeddings")\ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_pipeline = Pipeline(stages = [ documentAssembler, sbert_embedder, rxnorm_resolver]) model = rxnorm_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_model = LightPipeline(model) result = light_model.fullAnnotate(["Coumadin 5 mg", "aspirin", "avandia 4 mg"]) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm_augmented_cased", "en", "clinical/models") .setInputCols("sbert_embeddings") .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val data = Seq(Array("Coumadin 5 mg", "aspirin", ""avandia 4 mg")).toDS.toDF("text") val result= rxnorm_pipelineModel.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnen.med_ner.deid_subentityorm_augmented").predict("""Coumadin 5 mg""") ```

## Results ```bash | | RxNormCode | Resolution | all_k_results | all_k_distances | all_k_cosine_distances | all_k_resolutions | all_k_aux_labels | |---:|-------------:|:-----------------------------------------|:----------------------------------|:----------------------------------|:----------------------------------|:----------------------------------------------------------------|:----------------------------------| | 0 | 855333 | warfarin sodium 5 MG [Coumadin] | 855333:::432467:::438740:::103... | 3.0367:::4.7790:::4.7790:::5.3... | 0.0161:::0.0395:::0.0395:::0.0... | warfarin sodium 5 MG [Coumadin]:::coumarin 5 MG Oral Tablet:... | Branded Drug Comp:::Clinical D... | | 1 | 1537020 | aspirin Effervescent Oral Tablet | 1537020:::1191:::1295740:::405... | 0.0000:::0.0000:::4.1826:::5.7... | 0.0000:::0.0000:::0.0292:::0.0... | aspirin Effervescent Oral Tablet:::aspirin:::aspirin Oral Po... | Clinical Drug Form:::Ingredien... | | 2 | 261242 | rosiglitazone 4 MG Oral Tablet [Avandia] | 261242:::810073:::153845:::109... | 0.0000:::4.7482:::5.0125:::5.2... | 0.0000:::0.0365:::0.0409:::0.0... | rosiglitazone 4 MG Oral Tablet [Avandia]:::fesoterodine fuma... | Branded Drug:::Branded Drug Co... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_augmented| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|976.1 MB| |Case sensitive:|false| --- layout: model title: Fast Neural Machine Translation Model from Basque to English author: John Snow Labs name: opus_mt_eu_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, eu, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `eu` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_eu_en_xx_2.7.0_2.4_1609166590644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_eu_en_xx_2.7.0_2.4_1609166590644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_eu_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to French Pipeline author: John Snow Labs name: translate_en_fr date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, fr, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `fr` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_fr_xx_2.7.0_2.4_1609684801803.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_fr_xx_2.7.0_2.4_1609684801803.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_fr", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_fr", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.fr').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_fr| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Romanian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, ro, open_source] task: Embeddings language: ro edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ro_3.4.1_3.0_1647454014729.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ro_3.4.1_3.0_1647454014729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_hier_triplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_hier_triplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_hier_triplet_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_hier_triplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_hier_triplet_epochs_1_shard_1_squad2.0 --- layout: model title: Part of Speech for Korean author: John Snow Labs name: pos_ud_kaist date: 2021-03-09 tags: [part_of_speech, open_source, korean, pos_ud_kaist, ko] task: Part of Speech Tagging language: ko edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - CCONJ - ADV - SCONJ - DET - NOUN - VERB - ADJ - PUNCT - AUX - PRON - PROPN - NUM - INTJ - PART - X - ADP - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_kaist_ko_3.0.0_3.0_1615292391244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_kaist_ko_3.0.0_3.0_1615292391244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_kaist", "ko") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['John Snow Labs에서 안녕하세요! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_kaist", "ko") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("John Snow Labs에서 안녕하세요! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""John Snow Labs에서 안녕하세요! ""] token_df = nlu.load('ko.pos.ud_kaist').predict(text) token_df ```

## Results ```bash token pos 0 J NOUN 1 o NOUN 2 h NOUN 3 n SCONJ 4 S X 5 n X 6 o X 7 w X 8 L X 9 a X 10 b X 11 s X 12 에 ADP 13 서 SCONJ 14 안 ADV 15 녕 VERB 16 하세요 VERB 17 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_kaist| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|ko| --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from cj-mills) author: John Snow Labs name: xlmroberta_ner_cj_mills_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `cj-mills`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cj_mills_base_finetuned_panx_all_xx_4.1.0_3.0_1660427899930.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cj_mills_base_finetuned_panx_all_xx_4.1.0_3.0_1660427899930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cj_mills_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cj_mills_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_cj_mills_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|860.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/cj-mills/xlm-roberta-base-finetuned-panx-all --- layout: model title: Pipeline to Detect PHI for Generic Deidentification in Romanian (BERT) author: John Snow Labs name: ner_deid_generic_bert_pipeline date: 2023-03-09 tags: [licensed, clinical, ro, deidentification, phi, generic, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_generic_bert](https://nlp.johnsnowlabs.com/2022/11/22/ner_deid_generic_bert_ro.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_pipeline_ro_4.3.0_3.2_1678352946195.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_pipeline_ro_4.3.0_3.2_1678352946195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_generic_bert_pipeline", "ro", "clinical/models") text = '''Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui, 737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_generic_bert_pipeline", "ro", "clinical/models") val text = "Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui, 737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401" val result = pipeline.fullAnnotate(text) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265905","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265905","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265905").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265905| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265905 --- layout: model title: Fast Neural Machine Translation Model from English to Afro-Asiatic Languages author: John Snow Labs name: opus_mt_en_afa date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, afa, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `afa` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_afa_xx_2.7.0_2.4_1609169665906.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_afa_xx_2.7.0_2.4_1609169665906.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_afa| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts RxNorm Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_RXNORM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_3.0.4_3.0_1636395903630.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_3.0.4_3.0_1636395903630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 386165| 0.1567|hypercal:::hypersed:::hypertears:::hyperstat...|386165:::217667::...| |chronic renal ins...| 83|109| PROBLEM| 218689| 0.1036|nephro calci:::dialysis solutions:::creatini...|218689:::3310:::2...| | COPD| 113|116| PROBLEM|1539999| 0.1644|broncomar dm:::acne medication:::carbon mono...|1539999:::214981:...| | gastritis| 120|128| PROBLEM| 225965| 0.1983|gastroflux:::gastroflux oral product:::uceri...|225965:::1176661:...| | TIA| 136|138| PROBLEM|1089812| 0.0625|thera tears:::thiotepa injection:::nature's ...|1089812:::1660003...| |a non-ST elevatio...| 182|202| PROBLEM| 218767| 0.1007|non-aspirin pm:::aspirin-free:::non aspirin ...|218767:::215440::...| |Guaiac positive s...| 208|229| PROBLEM|1294361| 0.0820|anusol rectal product:::anusol hc rectal pro...|1294361:::1166715...| |cardiac catheteri...| 295|317| TEST| 385247| 0.1566|cardiacap:::cardiology pack:::cardizem:::car...|385247:::545063::...| | PTCA| 324|327|TREATMENT| 8410| 0.0867|alteplase:::reteplase:::pancuronium:::tripe ...|8410:::76895:::78...| | mid LAD lesion| 332|345| PROBLEM| 151672| 0.0549|dulcolax:::lazerformalyde:::linaclotide:::du...|151672:::217985::...| +--------------------+-----+---+---------+-------+----------+-----------------------------------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, drugs_sbert_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on November 2020 RxNorm Clinical Drugs ontology graph with ``sbiobert_base_cased_mli`` embeddings. https://www.nlm.nih.gov/pubs/techbull/nd20/brief/nd20_rx_norm_november_release.html --- layout: model title: Stopwords Remover for Marathi language (187 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, mr, open_source] task: Stop Words Removal language: mr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_mr_3.4.1_3.0_1646672300971.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_mr_3.4.1_3.0_1646672300971.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+ |result | +----------------------------+ |[माझ्यापेक्षा, चांगले, नाही]| +----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|mr| |Size:|2.1 KB| --- layout: model title: English T5ForConditionalGeneration Cased model (from ThomasNLG) author: John Snow Labs name: t5_qg_webnlg_synth date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-qg_webnlg_synth-en` is a English model originally trained by `ThomasNLG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_qg_webnlg_synth_en_4.3.0_3.0_1675125600977.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_qg_webnlg_synth_en_4.3.0_3.0_1675125600977.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_qg_webnlg_synth","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_qg_webnlg_synth","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_qg_webnlg_synth| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|280.8 MB| ## References - https://huggingface.co/ThomasNLG/t5-qg_webnlg_synth-en - https://github.com/ThomasScialom/QuestEval - https://arxiv.org/abs/2104.07555 --- layout: model title: English asr_wav2vec2_xlsr_53_phon TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_53_phon date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_53_phon` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_53_phon_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_53_phon_en_4.2.0_3.0_1664109509538.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_53_phon_en_4.2.0_3.0_1664109509538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_53_phon', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_53_phon", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_53_phon| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|756.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Hindi asr_Wav2Vec2_xls_r_lm_300m TFWav2Vec2ForCTC from LegolasTheElf author: John Snow Labs name: asr_Wav2Vec2_xls_r_lm_300m date: 2022-09-26 tags: [wav2vec2, hi, audio, open_source, asr] task: Automatic Speech Recognition language: hi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_xls_r_lm_300m` is a Hindi model originally trained by LegolasTheElf. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Wav2Vec2_xls_r_lm_300m_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_xls_r_lm_300m_hi_4.2.0_3.0_1664190519147.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_xls_r_lm_300m_hi_4.2.0_3.0_1664190519147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Wav2Vec2_xls_r_lm_300m", "hi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Wav2Vec2_xls_r_lm_300m", "hi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Wav2Vec2_xls_r_lm_300m| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|hi| |Size:|1.2 GB| --- layout: model title: Catalan RobertaForQuestionAnswering (from projecte-aina) author: John Snow Labs name: roberta_qa_roberta_base_ca_cased_qa date: 2022-06-20 tags: [ca, open_source, question_answering, roberta] task: Question Answering language: ca edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-ca-cased-qa` is a Catalan model originally trained by `projecte-aina`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_ca_cased_qa_ca_4.0.0_3.0_1655730281795.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_ca_cased_qa_ca_4.0.0_3.0_1655730281795.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_ca_cased_qa","ca") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_ca_cased_qa","ca") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_ca_cased_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|ca| |Size:|451.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/projecte-aina/roberta-base-ca-cased-qa - https://arxiv.org/abs/1907.11692 - https://github.com/projecte-aina/club --- layout: model title: Relation extraction between Drugs and ADE (ReDL) author: John Snow Labs name: redl_ade_biobert date: 2021-07-12 tags: [relation_extraction, en, clinical, licensed, ade, biobert] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.1.2 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is an end-to-end trained BioBERT model, capable of Relating Drugs and adverse reactions caused by them; It predicts if an adverse event is caused by a drug or not. `1` : Shows the adverse event and drug entities are related, `0` : Shows the adverse event and drug entities are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/RE_ADE.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_ade_biobert_en_3.1.2_3.0_1626105541347.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_ade_biobert_en_3.1.2_3.0_1626105541347.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |-----------:|:----------|----------------:|--------------:|:----------|:----------|----------------:|--------------:|:---------------|-------------:| | 1 | DRUG | 12 | 18 | Lipitor | ADE | 52 | 65 | severe fatigue | 0.998156 | | 1 | DRUG | 97 | 105 | voltarene | ADE | 144 | 156 | muscle cramps | 0.985513 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_ade_biobert| |Compatibility:|Healthcare NLP 3.1.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[redl_ner_chunks, document]| |Output Labels:|[relations]| |Language:|en| ## Data Source This model is trained on custom data annotated by JSL. ## Benchmarking ```bash label Recall Precision F1 Support 0 0.829 0.895 0.861 1146 1 0.955 0.923 0.939 2454 Avg. 0.892 0.909 0.900 - Weighted-Avg. 0.915 0.914 0.914 - ``` --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_marcel TFWav2Vec2ForCTC from marcel author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_german_by_marcel date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_marcel` is a German model originally trained by marcel. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_german_by_marcel_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_marcel_de_4.2.0_3.0_1664101884066.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_marcel_de_4.2.0_3.0_1664101884066.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_german_by_marcel", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_german_by_marcel", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_german_by_marcel| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-1024-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1657183896456.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1657183896456.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_1024_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-1024-finetuned-squad-seed-0 --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-32-finetuned-squad-seed-2` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2_en_4.0.0_3.0_1654191544720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2_en_4.0.0_3.0_1654191544720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_32d_seed_2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|376.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-32-finetuned-squad-seed-2 --- layout: model title: English BertForQuestionAnswering Tiny Cased model (from mrm8488) author: John Snow Labs name: bert_qa_tiny_wrslb_finetuned_squadv1 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-wrslb-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_tiny_wrslb_finetuned_squadv1_en_4.0.0_3.0_1657188696480.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_tiny_wrslb_finetuned_squadv1_en_4.0.0_3.0_1657188696480.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tiny_wrslb_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_tiny_wrslb_finetuned_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tiny_wrslb_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|16.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-tiny-wrslb-finetuned-squadv1 --- layout: model title: Fast Neural Machine Translation Model from English to Hindi author: John Snow Labs name: opus_mt_en_hi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, hi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `hi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_hi_xx_2.7.0_2.4_1609169231360.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_hi_xx_2.7.0_2.4_1609169231360.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_hi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Russian Named Entity Recognition (from IlyaGusev) author: John Snow Labs name: bert_ner_rubertconv_toxic_editor date: 2022-05-09 tags: [bert, ner, token_classification, ru, open_source] task: Named Entity Recognition language: ru edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `rubertconv_toxic_editor` is a Russian model orginally trained by `IlyaGusev`. ## Predicted Entities `equal`, `replace`, `delete`, `insert` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_rubertconv_toxic_editor_ru_3.4.2_3.0_1652099038495.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_rubertconv_toxic_editor_ru_3.4.2_3.0_1652099038495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_rubertconv_toxic_editor| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ru| |Size:|662.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/IlyaGusev/rubertconv_toxic_editor - https://colab.research.google.com/drive/1NUSO1QGlDgD-IWXa2SpeND089eVxrCJW - https://github.com/skoltech-nlp/russe_detox_2022/tree/main/data --- layout: model title: Aspect based Sentiment Analysis for restaurant reviews author: John Snow Labs name: ner_aspect_based_sentiment date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Automatically detect positive, negative and neutral aspects about restaurants from user reviews. Instead of labelling the entire review as negative or positive, this model helps identify which exact phrases relate to sentiment identified in the review. ## Predicted Entities `NEG`, `POS` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/ASPECT_BASED_SENTIMENT_RESTAURANT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_aspect_based_sentiment_en_3.0.0_3.0_1617209723737.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_aspect_based_sentiment_en_3.0.0_3.0_1617209723737.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+-------------------+-----------+ | sentence | aspect | sentiment | +----------------------------------------------------------------------------------------------------+-------------------+-----------+ | We loved our Thai-style main which amazing with lots of flavours very impressive for vegetarian. | Thai-style main | positive | | We loved our Thai-style main which amazing with lots of flavours very impressive for vegetarian. | lots of flavours | positive | | But the service was below average and the chips were too terrible to finish. | service | negative | | But the service was below average and the chips were too terrible to finish. | chips | negative | +----------------------------------------------------------------------------------------------------+-------------------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_aspect_based_sentiment| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, embeddings]| |Output Labels:|[absa]| |Language:|en| --- layout: model title: English asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6 TFWav2Vec2ForCTC from chrisvinsen author: John Snow Labs name: asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6` is a English model originally trained by chrisvinsen. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_en_4.2.0_3.0_1664106725822.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_en_4.2.0_3.0_1664106725822.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect posology entities (biobert) author: John Snow Labs name: ner_posology_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect Drug, Dosage and administration instructions in text using pretraiend NER model. ## Predicted Entities `FREQUENCY`, `DRUG`, `STRENGTH`, `FORM`, `DURATION`, `DOSAGE`, `ROUTE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_biobert_en_3.0.0_3.0_1617260806766.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_biobert_en_3.0.0_3.0_1617260806766.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_posology_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_posology_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology.biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash label precision recall f1-score support B-DOSAGE 0.78 0.67 0.72 559 B-DRUG 0.93 0.94 0.94 3865 B-DURATION 0.79 0.81 0.80 331 B-FORM 0.90 0.87 0.88 1472 B-FREQUENCY 0.92 0.94 0.93 1577 B-ROUTE 0.94 0.85 0.89 772 B-STRENGTH 0.88 0.92 0.90 2519 I-DOSAGE 0.62 0.57 0.60 357 I-DRUG 0.81 0.89 0.85 1539 I-DURATION 0.80 0.89 0.84 796 I-FORM 0.58 0.54 0.56 142 I-FREQUENCY 0.86 0.93 0.89 2424 I-ROUTE 1.00 0.47 0.64 32 I-STRENGTH 0.85 0.91 0.88 2972 O 0.98 0.98 0.98 101134 accuracy - - 0.97 120491 macro-avg 0.84 0.81 0.82 120491 weighted-avg 0.97 0.97 0.97 120491 ``` --- layout: model title: XLM-RoBERTa Base, CoNLL-03 NER Pipeline author: ahmedlone127 name: xlm_roberta_base_token_classifier_conll03_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, xlm_roberta, conll03, xlm, base, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/10/03/xlm_roberta_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/xlm_roberta_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655217192119.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/xlm_roberta_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655217192119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlm_roberta_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("xlm_roberta_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_ksponspeech_1_20", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_ksponspeech_1_20", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_ksponspeech_1_20| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_400000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-400000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_400000_cased_generator_de_3.4.4_3.0_1652786393218.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_400000_cased_generator_de_3.4.4_3.0_1652786393218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_400000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|223.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-400000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: Translate English to Azerbaijani Pipeline author: John Snow Labs name: translate_en_az date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, az, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `az` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_az_xx_2.7.0_2.4_1609685842780.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_az_xx_2.7.0_2.4_1609685842780.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_az", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_az", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.az').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_az| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Electronics And Electrical Engineering Document Classifier (EURLEX) author: John Snow Labs name: legclf_electronics_and_electrical_engineering_bert date: 2023-03-06 tags: [en, legal, classification, clauses, electronics_and_electrical_engineering, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_electronics_and_electrical_engineering_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Electronics_and_Electrical_Engineering or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Electronics_and_Electrical_Engineering`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_electronics_and_electrical_engineering_bert_en_1.0.0_3.0_1678111679765.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_electronics_and_electrical_engineering_bert_en_1.0.0_3.0_1678111679765.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_electronics_and_electrical_engineering_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Electronics_and_Electrical_Engineering]| |[Other]| |[Other]| |[Electronics_and_Electrical_Engineering]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_electronics_and_electrical_engineering_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Electronics_and_Electrical_Engineering 0.86 0.83 0.84 58 Other 0.85 0.88 0.86 64 accuracy - - 0.85 122 macro-avg 0.85 0.85 0.85 122 weighted-avg 0.85 0.85 0.85 122 ``` --- layout: model title: Legal Disclosure Of Information Clause Binary Classifier author: John Snow Labs name: legclf_disclosure_of_information_clause date: 2023-01-27 tags: [en, legal, classification, disclosure, information, clauses, disclosure_of_information, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `disclosure-of-information` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `disclosure-of-information`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_disclosure_of_information_clause_en_1.0.0_3.0_1674820995298.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_disclosure_of_information_clause_en_1.0.0_3.0_1674820995298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_disclosure_of_information_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[disclosure-of-information]| |[other]| |[other]| |[disclosure-of-information]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_disclosure_of_information_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support disclosure-of-information 1.00 0.96 0.98 26 other 0.97 1.00 0.99 39 accuracy - - 0.98 65 macro-avg 0.99 0.98 0.98 65 weighted-avg 0.99 0.98 0.98 65 ``` --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_jsl_rxnorm_cased embeddings) author: John Snow Labs name: sbiobertresolve_jsl_rxnorm_augmented date: 2021-12-27 tags: [en, clinical, entity_resolution, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbiobert_jsl_rxnorm_cased` Sentence Bert Embeddings. It is trained on the augmented version of the dataset which is used in previous RxNorm resolver models. Additionally, this model returns concept classes of the drugs in all_k_aux_labels column. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_jsl_rxnorm_augmented_en_3.3.4_2.4_1640637079907.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_jsl_rxnorm_augmented_en_3.3.4_2.4_1640637079907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_jsl_rxnorm_cased', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_jsl_rxnorm_augmented", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver]) light_model = LightPipeline(rxnorm_pipelineModel) result = light_model.fullAnnotate(["Coumadin 5 mg", "aspirin", "Neurontin 300", "avandia 4 mg"]) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_jsl_rxnorm_cased", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_jsl_rxnorm_augmented", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val light_model = LightPipeline(rxnorm_pipelineModel) val result = light_model.fullAnnotate(Array("Coumadin 5 mg", "aspirin", "avandia 4 mg")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.augmented").predict("""Coumadin 5 mg""") ```

## Results ```bash | | RxNormCode | Resolution | all_k_results | all_k_distances | all_k_cosine_distances | all_k_resolutions | all_k_aux_labels | |---:|-------------:|:-------------------------------------------------------------------|:----------------------------------|:----------------------------------|:----------------------------------|:----------------------------------------------------------------|:----------------------------------| | 0 | 855333 | warfarin sodium 5 MG [Coumadin] | 855333:::432467:::438740:::103... | 0.0000:::5.0617:::5.0617:::5.9... | 0.0000:::0.0388:::0.0388:::0.0... | warfarin sodium 5 MG [Coumadin]:::coumarin 5 MG Oral Tablet:... | Branded Drug Comp:::Clinical D... | | 1 | 1537020 | aspirin Effervescent Oral Tablet | 1537020:::1191:::405403:::2187... | 0.0000:::0.0000:::9.0615:::9.4... | 0.0000:::0.0000:::0.1268:::0.1... | aspirin Effervescent Oral Tablet:::aspirin:::YSP Aspirin:::N... | Clinical Drug Form:::Ingredien... | | 2 | 261242 | rosiglitazone 4 MG Oral Tablet [Avandia] | 261242:::208364:::1792373:::57... | 0.0000:::8.0227:::8.1631:::8.2... | 0.0000:::0.0982:::0.1001:::0.1... | rosiglitazone 4 MG Oral Tablet [Avandia]:::triamcinolone 4 M... | Branded Drug:::Branded Drug:::... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_jsl_rxnorm_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|970.8 MB| |Case sensitive:|false| --- layout: model title: Chinese BertForTokenClassification Base Cased model (from ckiplab) author: John Snow Labs name: bert_token_classifier_base_han_chinese_ws date: 2022-11-30 tags: [zh, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-han-chinese-ws` is a Chinese model originally trained by `ckiplab`. ## Predicted Entities `B`, `I` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_han_chinese_ws_zh_4.2.4_3.0_1669814901320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_han_chinese_ws_zh_4.2.4_3.0_1669814901320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_han_chinese_ws","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_han_chinese_ws","zh") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_base_han_chinese_ws| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|zh| |Size:|395.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ckiplab/bert-base-han-chinese-ws - https://github.com/ckiplab/han-transformers - http://lingcorpus.iis.sinica.edu.tw/cgi-bin/kiwi/akiwi/kiwi.sh - http://lingcorpus.iis.sinica.edu.tw/cgi-bin/kiwi/dkiwi/kiwi.sh - http://lingcorpus.iis.sinica.edu.tw/cgi-bin/kiwi/pkiwi/kiwi.sh - http://asbc.iis.sinica.edu.tw - https://ckip.iis.sinica.edu.tw/ --- layout: model title: English asr_model_sid_voxforge_cetuc_2 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: asr_model_sid_voxforge_cetuc_2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_sid_voxforge_cetuc_2` is a English model originally trained by joaoalvarenga. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_model_sid_voxforge_cetuc_2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_model_sid_voxforge_cetuc_2_en_4.2.0_3.0_1664022318789.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_model_sid_voxforge_cetuc_2_en_4.2.0_3.0_1664022318789.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_model_sid_voxforge_cetuc_2", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_model_sid_voxforge_cetuc_2", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_model_sid_voxforge_cetuc_2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect Drug Information (Small) author: John Snow Labs name: ner_posology date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for posology, this NER is trained with the ``embeddings_clinical`` word embeddings model, so be sure to use the same embeddings in the pipeline. ## Predicted Entities ``DOSAGE``, ``DRUG``, ``DURATION``, ``FORM``, ``FREQUENCY``, ``ROUTE``, ``STRENGTH``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_en_3.0.0_2.4_1617208445872.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_en_3.0.0_2.4_1617208445872.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_posology_small","en","clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val model = MedicalNerModel.pretrained("ner_posology_small","en","clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, model, ner_converter)) val data = Seq("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunk |ner | +--------------+---------+ |insulin |DRUG | |Bactrim |DRUG | |for 14 days |DURATION | |Fragmin |DRUG | |5000 units |DOSAGE | |subcutaneously|ROUTE | |daily |FREQUENCY| |Xenaderm |DRUG | |topically |ROUTE | |b.i.d., |FREQUENCY| |Lantus |DRUG | |40 units |DOSAGE | |subcutaneously|ROUTE | |at bedtime |FREQUENCY| |OxyContin |DRUG | |30 mg |STRENGTH | |p.o |ROUTE | |q.12 h |FREQUENCY| |folic acid |DRUG | |1 mg |STRENGTH | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on the 2018 i2b2 dataset (no FDA) with ``embeddings_clinical``. https://www.i2b2.org/NLP/Medication ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | B-DRUG | 1408 | 62 | 99 | 0.957823 | 0.934307 | 0.945919 | | 1 | B-STRENGTH | 470 | 43 | 29 | 0.916179 | 0.941884 | 0.928854 | | 2 | I-DURATION | 123 | 22 | 8 | 0.848276 | 0.938931 | 0.891304 | | 3 | I-STRENGTH | 499 | 66 | 15 | 0.883186 | 0.970817 | 0.924931 | | 4 | I-FREQUENCY | 945 | 47 | 55 | 0.952621 | 0.945 | 0.948795 | | 5 | B-FORM | 365 | 13 | 12 | 0.965608 | 0.96817 | 0.966887 | | 6 | B-DOSAGE | 298 | 27 | 26 | 0.916923 | 0.919753 | 0.918336 | | 7 | I-DOSAGE | 348 | 29 | 22 | 0.923077 | 0.940541 | 0.931727 | | 8 | I-DRUG | 208 | 25 | 60 | 0.892704 | 0.776119 | 0.830339 | | 9 | I-ROUTE | 10 | 0 | 2 | 1 | 0.833333 | 0.909091 | | 10 | B-ROUTE | 467 | 4 | 25 | 0.991507 | 0.949187 | 0.969886 | | 11 | B-DURATION | 64 | 10 | 10 | 0.864865 | 0.864865 | 0.864865 | | 12 | B-FREQUENCY | 588 | 12 | 17 | 0.98 | 0.971901 | 0.975934 | | 13 | I-FORM | 264 | 5 | 4 | 0.981413 | 0.985075 | 0.98324 | | 14 | Macro-average | 6057 | 365 | 384 | 0.93387 | 0.924277 | 0.929049 | | 15 | Micro-average | 6057 | 365 | 384 | 0.943164 | 0.940382 | 0.941771 | ``` --- layout: model title: Legal Performance Clause Binary Classifier author: John Snow Labs name: legclf_performance_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `performance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `performance` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_performance_clause_en_1.0.0_3.2_1660123818700.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_performance_clause_en_1.0.0_3.2_1660123818700.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_performance_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[performance]| |[other]| |[other]| |[performance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_performance_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.90 1.00 0.95 89 performance 1.00 0.74 0.85 39 accuracy - - 0.92 128 macro-avg 0.95 0.87 0.90 128 weighted-avg 0.93 0.92 0.92 128 ``` --- layout: model title: Translate English to Tsonga Pipeline author: John Snow Labs name: translate_en_ts date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ts, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ts` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ts_xx_2.7.0_2.4_1609699062996.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ts_xx_2.7.0_2.4_1609699062996.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ts", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ts", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ts').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ts| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Finnish BERT Embeddings (Base Uncased) author: John Snow Labs name: bert_finnish_uncased date: 2020-08-31 task: Embeddings language: fi edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, fi] supported: true deprecated: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A version of Google's BERT deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. `FinBERT` features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words. `FinBERT` has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train `FinBERT`. These features allow `FinBERT` to outperform not only Multilingual BERT but also all previously proposed models when fine-tuned for Finnish natural language processing tasks. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_finnish_uncased_fi_2.6.0_2.4_1598897239983.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_finnish_uncased_fi_2.6.0_2.4_1598897239983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_finnish_uncased", "fi") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Rakastan NLP: tä']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_finnish_uncased", "fi") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("Rakastan NLP: tä").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Rakastan NLP: tä"] embeddings_df = nlu.load('fi.embed.bert.uncased.').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token fi_embed_bert_uncased__embeddings Rakastan [-0.5126021504402161, -1.1741008758544922, 0.6... NLP [1.4763829708099365, -1.5427947044372559, 0.80... : [-0.2581554353237152, -0.5670831203460693, -1.... tä [0.39770740270614624, -0.7221324443817139, 0.1... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_finnish_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[fi]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://github.com/TurkuNLP/FinBERT --- layout: model title: English asr_sanskrit TFWav2Vec2ForCTC from Tarakki100 author: John Snow Labs name: asr_sanskrit date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_sanskrit` is a English model originally trained by Tarakki100. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_sanskrit_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_sanskrit_en_4.2.0_3.0_1664112373546.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_sanskrit_en_4.2.0_3.0_1664112373546.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_sanskrit", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_sanskrit", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_sanskrit| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: Pre-trained Pipeline for Few-NERD NER Model author: John Snow Labs name: nerdl_fewnerd_subentity_100d_pipeline date: 2022-06-28 tags: [fewnerd, ner, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on Few-NERD/inter public dataset and it extracts 66 entities that are in general scope. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_pipeline_en_4.0.0_3.0_1656388795031.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_pipeline_en_4.0.0_3.0_1656388795031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python fewnerd_pipeline = PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") fewnerd_pipeline.annotate("""12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.""") ``` ```scala val pipeline = new PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") val result = pipeline.fullAnnotate("12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.")(0) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_pubmed| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|468.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/raynardj/roberta-pubmed - https://pubmed.ncbi.nlm.nih.gov/ - https://www.ncbi.nlm.nih.gov/mesh/ --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from cosmo) author: John Snow Labs name: distilbert_qa_cosmo_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `cosmo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cosmo_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770516755.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cosmo_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770516755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cosmo_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cosmo_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cosmo_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/cosmo/distilbert-base-uncased-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Catalan author: John Snow Labs name: opus_mt_en_ca date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ca, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ca` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ca_xx_2.7.0_2.4_1609167744249.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ca_xx_2.7.0_2.4_1609167744249.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ca| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from skandaonsolve) author: John Snow Labs name: roberta_qa_finetuned_location date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-location` is a English model originally trained by `skandaonsolve`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_location_en_4.3.0_3.0_1674220382399.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_location_en_4.3.0_3.0_1674220382399.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_location","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_location","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_location| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-location --- layout: model title: Finnish Legal Roberta Embeddings author: John Snow Labs name: roberta_base_finnish_legal date: 2023-02-16 tags: [fi, finnish, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: fi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-finnish-roberta-base` is a Finnish model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_finnish_legal_fi_4.2.4_3.0_1676561071432.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_finnish_legal_fi_4.2.4_3.0_1676561071432.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_finnish_legal", "fi")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_finnish_legal", "fi") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_finnish_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fi| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-finnish-roberta-base --- layout: model title: Stopwords Remover for Dutch language (352 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, nl, open_source] task: Stop Words Removal language: nl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_nl_3.4.1_3.0_1646673228420.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_nl_3.4.1_3.0_1646673228420.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[] | +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|nl| |Size:|2.4 KB| --- layout: model title: Finance Pipeline (Headers / Subheaders) author: John Snow Labs name: finpipe_header_subheader date: 2023-01-20 tags: [en, finance, ner, licensed, contextual_parser] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a finance pretrained pipeline that will help you split long financial documents into smaller sections. To do that, it detects Headers and Subheaders of different sections. You can then use the beginning and end information in the metadata to retrieve the text between those headers. PART I, PART II, etc are HEADERS Item 1, Item 2, etc are also HEADERS Item 1A, 2B, etc are SUBHEADERS 1., 2., 2.1, etc. are SUBHEADERS {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finpipe_header_subheader_en_1.0.0_3.0_1674243435691.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finpipe_header_subheader_en_1.0.0_3.0_1674243435691.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python finance_pipeline = nlp.PretrainedPipeline("finpipe_header_subheader", "en", "finance/models") text = [""" Item 2. Definitions. For purposes of this Agreement, the following terms have the meanings ascribed thereto in this Section 1. 2. Appointment as Reseller. Item 2A. Appointment. The Company hereby [***]. Allscripts may also disclose Company's pricing information relating to its Merchant Processing Services and facilitate procurement of Merchant Processing Services on behalf of Sublicensed Customers, including, without limitation by references to such pricing information and Merchant Processing Services in Customer Agreements. 6 Item 2B. Customer Agreements."""] result = finance_pipeline.annotate(text) ```

## Results ```bash | chunks | begin | end | entities | |------------------------------:|------:|----:|----------:| | Item 2. Definitions. | 1 | 21 | HEADER | | Item 2A. Appointment. | 158 | 179 | SUBHEADER | | Item 2B. Customer Agreements. | 538 | 566 | SUBHEADER | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finpipe_header_subheader| |Type:|pipeline| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|23.6 KB| ## Included Models - DocumentAssembler - TokenizerModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel --- layout: model title: English asr_wav2vec2_base_timit_demo_by_patrickvonplaten TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_by_patrickvonplaten` is a English model originally trained by patrickvonplaten. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten_en_4.2.0_3.0_1664025434108.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten_en_4.2.0_3.0_1664025434108.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_by_patrickvonplaten| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: xlmroberta_pos_xlm_roberta_base_english_upos date: 2022-05-18 tags: [xlm_roberta, pos, part_of_speech, en, open_source] task: Part of Speech Tagging language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-english-upos` is a English model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_pos_xlm_roberta_base_english_upos_en_3.4.2_3.0_1652837577026.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_pos_xlm_roberta_base_english_upos_en_3.4.2_3.0_1652837577026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_pos_xlm_roberta_base_english_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|791.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/xlm-roberta-base-english-upos - https://github.com/UniversalDependencies/UD_English-EWT - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Translate Kinyarwanda to English Pipeline author: John Snow Labs name: translate_rw_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, rw, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `rw` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_rw_en_xx_2.7.0_2.4_1609687466102.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_rw_en_xx_2.7.0_2.4_1609687466102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_rw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_rw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.rw.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_rw_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish RobertaForQuestionAnswering Large Cased model (from stevemobs) author: John Snow Labs name: roberta_qa_large_fine_tuned_squad date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-fine-tuned-squad-es` is a Spanish model originally trained by `stevemobs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_fine_tuned_squad_es_4.3.0_3.0_1674221753097.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_fine_tuned_squad_es_4.3.0_3.0_1674221753097.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_fine_tuned_squad","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_fine_tuned_squad","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_fine_tuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/stevemobs/roberta-large-fine-tuned-squad-es --- layout: model title: English asr_wav2vec2_xls_r_300m_cv8 TFWav2Vec2ForCTC from comodoro author: John Snow Labs name: asr_wav2vec2_xls_r_300m_cv8 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_cv8` is a English model originally trained by comodoro. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_cv8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_cv8_en_4.2.0_3.0_1664036662517.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_cv8_en_4.2.0_3.0_1664036662517.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_cv8", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_cv8", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_cv8| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Fake News Classifier author: John Snow Labs name: classifierdl_use_fakenews date: 2021-01-09 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, en, classifier] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Determine if news articles are Real or Fake. ## Predicted Entities `REAL`, `FAKE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_EN_FAKENEWS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_EN_FAKENEWS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_fakenews_en_2.7.1_2.4_1610187399147.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_fakenews_en_2.7.1_2.4_1610187399147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------------------------------------------------+------------+ |document |class | +--------------------------------------------------------------------------------------------------------+------------+ |Donald Trump a KGB Spy? 11/02/2016 In today’s video, Christopher Greene of AMTV reports Hillary Clinton | FAKE | +--------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_fakenews| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|tfhub_use| ## Data Source This model is trained on the fake new classification challenge. https://raw.githubusercontent.com/joolsa/fake_real_news_dataset/master/fake_or_real_news.csv.zip ## Benchmarking ```bash precision recall f1-score support FAKE 0.86 0.89 0.88 626 REAL 0.89 0.86 0.87 634 accuracy 0.87 1260 macro avg 0.88 0.87 0.87 1260 weighted avg 0.88 0.87 0.87 1260 ``` --- layout: model title: Fast Neural Machine Translation Model from Chuukese to English author: John Snow Labs name: opus_mt_chk_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, chk, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `chk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_chk_en_xx_2.7.0_2.4_1609169397835.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_chk_en_xx_2.7.0_2.4_1609169397835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_chk_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman` is a Finnish model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_fi_4.2.0_3.0_1664041894280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_fi_4.2.0_3.0_1664041894280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from deepset) author: John Snow Labs name: bert_qa_minilm_uncased_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `minilm-uncased-squad2` is a English model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_minilm_uncased_squad2_en_4.0.0_3.0_1654188279232.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_minilm_uncased_squad2_en_4.0.0_3.0_1654188279232.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_minilm_uncased_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_minilm_uncased_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.mini_lm_base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_minilm_uncased_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|123.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/deepset/minilm-uncased-squad2 - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py - https://twitter.com/deepset_ai - http://www.deepset.ai/jobs - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/ - https://deepset.ai/german-bert - https://www.linkedin.com/company/deepset-ai/ - https://github.com/deepset-ai/FARM - https://deepset.ai/germanquad --- layout: model title: Abkhazian asr_xls_r_ab_test_by_muneson TFWav2Vec2ForCTC from muneson author: John Snow Labs name: pipeline_asr_xls_r_ab_test_by_muneson date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_muneson` is a Abkhazian model originally trained by muneson. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xls_r_ab_test_by_muneson_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_ab_test_by_muneson_ab_4.2.0_3.0_1664019208833.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_ab_test_by_muneson_ab_4.2.0_3.0_1664019208833.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_r_ab_test_by_muneson', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_r_ab_test_by_muneson", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_r_ab_test_by_muneson| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|452.2 KB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from sasuke) author: John Snow Labs name: distilbert_qa_sasuke_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `sasuke`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sasuke_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772441119.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sasuke_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772441119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sasuke_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sasuke_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sasuke_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sasuke/distilbert-base-uncased-finetuned-squad --- layout: model title: Financial Deidentification Pipeline author: John Snow Labs name: finpipe_deid date: 2023-02-27 tags: [deid, deidentification, anonymization, en, licensed] task: [De-identification, Pipeline Finance] language: en edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true recommended: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Pretrained Pipeline aimed to deidentify legal and financial documents to be compliant with data privacy regulations as GDPR and CCPA. Since the models used in this pipeline are statistical, make sure you use this model in a human-in-the-loop process to guarantee a 100% accuracy. You can carry out both masking and obfuscation with this pipeline, on the following entities: `ALIAS`, `EMAIL`, `PHONE`, `PROFESSION`, `ORG`, `DATE`, `PERSON`, `ADDRESS`, `STREET`, `CITY`, `STATE`, `ZIP`, `COUNTRY`, `TITLE_CLASS`, `TICKER`, `STOCK_EXCHANGE`, `CFN`, `IRS` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/DEID_FIN/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finpipe_deid_en_1.0.0_3.0_1677508149273.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finpipe_deid_en_1.0.0_3.0_1677508149273.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("finpipe_deid", "en", "finance/models") sample = """CARGILL, INCORPORATED By: Pirkko Suominen Name: Pirkko Suominen Title: Director, Bio Technology Development, Date: 10/19/2011 BIOAMBER, SAS By: Jean-François Huc Name: Jean-François Huc Title: President Date: October 15, 2011 email : jeanfran@gmail.com phone : 1808733909 """ result = deid_pipeline.annotate(sample) print("\nMasked with entity labels") print("-"*30) print("\n".join(result['deidentified'])) print("\nMasked with chars") print("-"*30) print("\n".join(result['masked_with_chars'])) print("\nMasked with fixed length chars") print("-"*30) print("\n".join(result['masked_fixed_length_chars'])) print("\nObfuscated") print("-"*30) print("\n".join(result['obfuscated'])) ```

## Results ```bash Masked with entity labels ------------------------------ , By: Name: : , Date: , By: Name: : Date: email : phone : Masked with chars ------------------------------ [*****], [**********] By: [*************] Name: [*******************]: [**********************************] Center, Date: [********] [******], [*] By: [***************] Name: [**********************]: [*******]Date: [**************] email : [****************] phone : [********] Masked with fixed length chars ------------------------------ ****, **** By: **** Name: ****: ****, Date: **** ****, **** By: **** Name: ****: ****Date: **** email : **** phone : **** Obfuscated ------------------------------ MGT Trust Company, LLC., Clarus llc. By: Benjamin Dean Name: John Snow Labs Inc: Sales Manager, Date: 03/08/2025 Clarus llc., SESA CO. By: JAMES TURNER Name: MGT Trust Company, LLC.: Business ManagerDate: 11/7/2016 email : Tyrus@google.com phone : 78 834 854 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finpipe_deid| |Type:|pipeline| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|458.6 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - BertEmbeddings - FinanceNerModel - NerConverterInternalModel - FinanceNerModel - NerConverterInternalModel - FinanceNerModel - NerConverterInternalModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel --- layout: model title: Detect Clinical Conditions (ner_eu_clinical_condition - it) author: John Snow Labs name: ner_eu_clinical_condition date: 2023-02-06 tags: [it, clinical, licensed, ner, clinical_condition] task: Named Entity Recognition language: it edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition (NER) deep learning model for extracting clinical conditions from Italian texts. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nichols, Named Entity Recognition with Bidirectional LSTM-CNN. The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Predicted Entities `clinical_condition` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_it_4.2.8_3.0_1675726754516.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_it_4.2.8_3.0_1675726754516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","it")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_eu_clinical_condition', "it", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""Donna, 64 anni, ricovero per dolore epigastrico persistente, irradiato a barra e posteriormente, associato a dispesia e anoressia. Poche settimane dopo compaiono, però, iperemia, intenso edema vulvare ed una esione ulcerativa sul lato sinistro della parete rettale che la RM mostra essere una fistola transfinterica. Questi trattamenti determinano un miglioramento dell’ infiammazione ed una riduzione dell’ ulcera, ma i condilomi permangono inalterati."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","it") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_eu_clinical_condition", "it", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""Donna, 64 anni, ricovero per dolore epigastrico persistente, irradiato a barra e posteriormente, associato a dispesia e anoressia. Poche settimane dopo compaiono, però, iperemia, intenso edema vulvare ed una esione ulcerativa sul lato sinistro della parete rettale che la RM mostra essere una fistola transfinterica. Questi trattamenti determinano un miglioramento dell’ infiammazione ed una riduzione dell’ ulcera, ma i condilomi permangono inalterati.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------+------------------+ |chunk |ner_label | +----------------------+------------------+ |dolore epigastrico |clinical_condition| |anoressia |clinical_condition| |iperemia |clinical_condition| |edema |clinical_condition| |fistola transfinterica|clinical_condition| |infiammazione |clinical_condition| +----------------------+------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_condition| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|it| |Size:|903.5 KB| ## References The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Benchmarking ```bash label tp fp fn total precision recall f1 clinical_condition 208.0 35.0 46.0 254.0 0.8560 0.8189 0.8370 macro - - - - - - 0.8370 micro - - - - - - 0.8370 ``` --- layout: model title: English RobertaForQuestionAnswering (from saburbutt) author: John Snow Labs name: roberta_qa_roberta_large_tweetqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta_large_tweetqa` is a English model originally trained by `saburbutt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_tweetqa_en_4.0.0_3.0_1655739122606.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_tweetqa_en_4.0.0_3.0_1655739122606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_tweetqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_tweetqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.trivia.roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_tweetqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saburbutt/roberta_large_tweetqa --- layout: model title: English image_classifier_vit_iiif_manuscript_ ViTForImageClassification from davanstrien author: John Snow Labs name: image_classifier_vit_iiif_manuscript_ date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_iiif_manuscript_` is a English model originally trained by davanstrien. ## Predicted Entities `3rd upper flyleaf verso`, `Blank leaf recto`, `3rd lower flyleaf verso`, `2nd lower flyleaf verso`, `2nd upper flyleaf verso`, `flyleaf`, `1st upper flyleaf verso`, `1st lower flyleaf verso`, `fol`, `cover`, `Lower flyleaf verso`, `Blank leaf verso`, `Upper flyleaf verso` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_iiif_manuscript__en_4.1.0_3.0_1660170321999.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_iiif_manuscript__en_4.1.0_3.0_1660170321999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_iiif_manuscript_", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_iiif_manuscript_", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_iiif_manuscript_| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Purchase and sale Clause Binary Classifier author: John Snow Labs name: legclf_purchase_and_sale_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `purchase-and-sale` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `purchase-and-sale` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_purchase_and_sale_clause_en_1.0.0_3.2_1660123863834.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_purchase_and_sale_clause_en_1.0.0_3.2_1660123863834.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_purchase_and_sale_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[purchase-and-sale]| |[other]| |[other]| |[purchase-and-sale]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_purchase_and_sale_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.93 1.00 0.96 65 purchase-and-sale 1.00 0.88 0.94 43 accuracy - - 0.95 108 macro-avg 0.96 0.94 0.95 108 weighted-avg 0.96 0.95 0.95 108 ``` --- layout: model title: Detect Problems, Tests and Treatments (ner_clinical) in German author: John Snow Labs name: ner_clinical date: 2023-05-05 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terms in German. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `PROBLEM`, `TEST`, `TREATMENT` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/14.German_Healthcare_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_de_4.4.0_3.0_1683310968546.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_de_4.4.0_3.0_1683310968546.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "de", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) sample_text= """Verschlechterung von Schmerzen oder Schwäche in den Beinen , Verlust der Darm - oder Blasenfunktion oder andere besorgniserregende Symptome. Der Patient erhielt empirisch Ampicillin , Gentamycin und Flagyl sowie Narcan zur Umkehrung von Fentanyl . ALT war 181 , AST war 156 , LDH war 336 , alkalische Phosphatase war 214 und Bilirubin war insgesamt 12,7 .""" results = model.transform(spark.createDataFrame([[sample_text]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_clinical", "de", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""Verschlechterung von Schmerzen oder Schwäche in den Beinen , Verlust der Darm - oder Blasenfunktion oder andere besorgniserregende Symptome. Der Patient erhielt empirisch Ampicillin , Gentamycin und Flagyl sowie Narcan zur Umkehrung von Fentanyl . ALT war 181 , AST war 156 , LDH war 336 , alkalische Phosphatase war 214 und Bilirubin war insgesamt 12,7 .""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Schmerzen |PROBLEM | |Schwäche in den Beinen|PROBLEM | |Verlust der Darm |PROBLEM | |Blasenfunktion |PROBLEM | |Symptome |PROBLEM | |empirisch Ampicillin |TREATMENT| |Gentamycin |TREATMENT| |Flagyl |TREATMENT| |Narcan |TREATMENT| |Fentanyl |TREATMENT| |ALT |TEST | |AST |TEST | |LDH |TEST | |alkalische Phosphatase|TEST | |Bilirubin |TEST | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| |Size:|2.1 MB| ## Benchmarking ```bash label precision recall f1-score support B-PROBLEM 0.78 0.77 0.77 407 B-TEST 0.77 0.92 0.84 220 B-TREATMENT 0.82 0.71 0.76 241 I-PROBLEM 0.87 0.78 0.82 386 I-TEST 0.66 0.93 0.77 57 I-TREATMENT 0.68 0.76 0.72 76 O 0.96 0.97 0.96 4323 accuracy - - 0.92 5710 macro-avg 0.79 0.83 0.81 5710 weighted-avg 0.92 0.92 0.92 5710 ``` --- layout: model title: Javanese Bert Embeddings (Small, Imdb) author: John Snow Labs name: bert_embeddings_javanese_bert_small_imdb date: 2022-04-11 tags: [bert, embeddings, jv, open_source] task: Embeddings language: jv edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `javanese-bert-small-imdb` is a Javanese model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_bert_small_imdb_jv_3.4.2_3.0_1649676688860.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_bert_small_imdb_jv_3.4.2_3.0_1649676688860.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_javanese_bert_small_imdb| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|410.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-bert-small-imdb - https://arxiv.org/abs/1810.04805 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: Legal Bankruptcy Clause Binary Classifier author: John Snow Labs name: legclf_bankruptcy_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `bankruptcy` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `bankruptcy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_bankruptcy_clause_en_1.0.0_3.2_1660122157662.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_bankruptcy_clause_en_1.0.0_3.2_1660122157662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_bankruptcy_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[bankruptcy]| |[other]| |[other]| |[bankruptcy]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_bankruptcy_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support bankruptcy 1.00 0.73 0.84 26 other 0.94 1.00 0.97 107 accuracy - - 0.95 133 macro-avg 0.97 0.87 0.91 133 weighted-avg 0.95 0.95 0.94 133 ``` --- layout: model title: Detect PHI for Deidentification purposes (Spanish, Roberta, augmented) author: John Snow Labs name: ner_deid_subentity_roberta_augmented date: 2022-02-16 tags: [deid, es, licensed,clinical] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities, which is more than the previously released `ner_deid_subentity_roberta` model. This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf and MeddoCan datasets, and includes several data augmentation mechanisms. This is a version that includes Roberta Clinical embeddings. You can find as well `ner_deid_subentity_augmented` that uses Sciwi 300d embeddings based instead of Roberta. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `ID`, `STREET`, `USERNAME`, `SEX`, `EMAIL`, `ZIP`, `MEDICALRECORD`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_roberta_augmented_es_3.3.4_3.0_1645006804071.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_roberta_augmented_es_3.3.4_3.0_1645006804071.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") roberta_embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_subentity_roberta_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") nlpPipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] df = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(df).transform(df) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val roberta_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_deid_subentity_roberta_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val df = Seq(text).toDF("text") val results = pipeline.fit(df).transform(df) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.subentity.roberta").predict(""" Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-PATIENT| | Miguel| I-PATIENT| | Martínez| I-PATIENT| | ,| O| | varón| B-SEX| | de| O| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-CITY| | ,| O| | España| B-COUNTRY| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19| O| | el| O| | dia| O| | 14| B-DATE| | de| I-DATE| | Marzo| I-DATE| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| O| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-HOSPITAL| | San| I-HOSPITAL| | Carlos| I-HOSPITAL| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_roberta_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|16.3 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 1874.0 165.0 186.0 2060.0 0.9191 0.9097 0.9144 HOSPITAL 241.0 19.0 54.0 295.0 0.9269 0.8169 0.8685 DATE 954.0 17.0 15.0 969.0 0.9825 0.9845 0.9835 ORGANIZATION 2521.0 483.0 468.0 2989.0 0.8392 0.8434 0.8413 CITY 1464.0 369.0 289.0 1753.0 0.7987 0.8351 0.8165 ID 35.0 1.0 0.0 35.0 0.9722 1.0 0.9859 STREET 194.0 8.0 6.0 200.0 0.9604 0.97 0.9652 USERNAME 7.0 0.0 4.0 11.0 1.0 0.6364 0.7778 SEX 618.0 9.0 9.0 627.0 0.9856 0.9856 0.9856 EMAIL 134.0 0.0 0.0 134.0 1.0 1.0 1.0 ZIP 138.0 0.0 1.0 139.0 1.0 0.9928 0.9964 MEDICALRECORD 29.0 10.0 0.0 29.0 0.7436 1.0 0.8529 PROFESSION 231.0 20.0 30.0 261.0 0.9203 0.8851 0.9023 PHONE 44.0 0.0 6.0 50.0 1.0 0.88 0.9362 COUNTRY 458.0 96.0 103.0 561.0 0.8267 0.8164 0.8215 DOCTOR 432.0 38.0 48.0 480.0 0.9191 0.9 0.9095 AGE 509.0 9.0 10.0 519.0 0.9826 0.9807 0.9817 macro - - - - - - 0.9141 micro - - - - - - 0.8891 ``` --- layout: model title: Romanian T5ForConditionalGeneration Small Cased model (from BlackKakapo) author: John Snow Labs name: t5_small_grammar_v2 date: 2023-01-31 tags: [ro, open_source, t5, tensorflow] task: Text Generation language: ro edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-grammar-ro-v2` is a Romanian model originally trained by `BlackKakapo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_grammar_v2_ro_4.3.0_3.0_1675126347481.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_grammar_v2_ro_4.3.0_3.0_1675126347481.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_grammar_v2","ro") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_grammar_v2","ro") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_grammar_v2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ro| |Size:|288.1 MB| ## References - https://huggingface.co/BlackKakapo/t5-small-grammar-ro-v2 - https://img.shields.io/badge/V.2-06.08.2022-brightgreen --- layout: model title: Translate Lushai to English Pipeline author: John Snow Labs name: translate_lus_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, lus, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `lus` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_lus_en_xx_2.7.0_2.4_1609686287332.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_lus_en_xx_2.7.0_2.4_1609686287332.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_lus_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_lus_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.lus.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_lus_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English AlbertForQuestionAnswering model (from sultan) author: John Snow Labs name: albert_qa_BioM_xxlarge_SQuAD2 date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BioM-ALBERT-xxlarge-SQuAD2` is a English model originally trained by `sultan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_BioM_xxlarge_SQuAD2_en_4.0.0_3.0_1656063644904.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_BioM_xxlarge_SQuAD2_en_4.0.0_3.0_1656063644904.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_BioM_xxlarge_SQuAD2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_BioM_xxlarge_SQuAD2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.albert.xxl.by_sultan").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_BioM_xxlarge_SQuAD2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|771.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sultan/BioM-ALBERT-xxlarge-SQuAD2 - http://participants-area.bioasq.org/results/9b/phaseB/ - https://github.com/salrowili/BioM-Transformers --- layout: model title: Italian BertForMaskedLM Base Uncased model (from dbmdz) author: John Snow Labs name: bert_embeddings_base_italian_xxl_uncased date: 2022-12-02 tags: [it, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-italian-xxl-uncased` is a Italian model originally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_xxl_uncased_it_4.2.4_3.0_1670018034736.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_xxl_uncased_it_4.2.4_3.0_1670018034736.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_xxl_uncased","it") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_xxl_uncased","it") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_italian_xxl_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|415.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-xxl-uncased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Legal Registration Clause Binary Classifier author: John Snow Labs name: legclf_registration_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `registration` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `registration` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_registration_clause_en_1.0.0_3.2_1660123909144.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_registration_clause_en_1.0.0_3.2_1660123909144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_registration_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[registration]| |[other]| |[other]| |[registration]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_registration_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.90 0.97 0.93 65 registration-rights 0.94 0.83 0.88 41 accuracy - - 0.92 106 macro-avg 0.92 0.90 0.91 106 weighted-avg 0.92 0.92 0.91 106 ``` --- layout: model title: Pipeline to Detect Chemical Compounds and Genes author: John Snow Labs name: ner_chemprot_clinical_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chemprot_clinical](https://nlp.johnsnowlabs.com/2021/03/31/ner_chemprot_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_clinical_pipeline_en_4.3.0_3.2_1678865440862.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_clinical_pipeline_en_4.3.0_3.2_1678865440862.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_chemprot_clinical_pipeline", "en", "clinical/models") text = '''Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_chemprot_clinical_pipeline", "en", "clinical/models") val text = "Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemprot_clinical.pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:------------|-------------:| | 0 | Keratinocyte | 0 | 11 | GENE-Y | 0.7433 | | 1 | growth | 13 | 18 | GENE-Y | 0.6481 | | 2 | factor | 20 | 25 | GENE-Y | 0.5693 | | 3 | acidic | 31 | 36 | GENE-Y | 0.5518 | | 4 | fibroblast | 38 | 47 | GENE-Y | 0.5111 | | 5 | growth | 49 | 54 | GENE-Y | 0.4559 | | 6 | factor | 56 | 61 | GENE-Y | 0.5213 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemprot_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from deepset) author: John Snow Labs name: roberta_qa_large_squad2_hp date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad2-hp` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_squad2_hp_en_4.3.0_3.0_1674222219863.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_squad2_hp_en_4.3.0_3.0_1674222219863.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_squad2_hp","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_squad2_hp","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_squad2_hp| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/deepset/roberta-large-squad2-hp --- layout: model title: Summarize Clinical Question Notes author: John Snow Labs name: summarizer_clinical_questions date: 2023-04-03 tags: [licensed, en, clinical, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based summarization model that is finetuned with medical questions exchanged in clinical mediums (clinic, email, call center etc.) by John Snow Labs. It can generate summaries up to 512 tokens given an input text (max 1024 tokens). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/MEDICAL_TEXT_SUMMARIZATION_QA/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/32.Medical_Text_Summarization.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_en_4.3.2_3.0_1680550227628.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_en_4.3.2_3.0_1680550227628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['What are the treatments for hyperthyroidism?'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_questions| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.0 MB| --- layout: model title: Word2Vec Embeddings in Dimli (individual language) (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, diq, open_source] task: Embeddings language: diq edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_diq_3.4.1_3.0_1647467748159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_diq_3.4.1_3.0_1647467748159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_sundanese', lang = 'su') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_sundanese", lang = "su") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_sundanese| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|su| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Professions & Occupations NER model in Spanish (meddroprof_scielowiki) author: John Snow Labs name: meddroprof_scielowiki date: 2021-07-26 tags: [ner, licensed, prefessions, es, occupations] task: Named Entity Recognition language: es edition: Healthcare NLP 3.1.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description NER model that detects professions and occupations in Spanish texts. Trained with the `embeddings_scielowiki_300d` embeddings, and the same `WordEmbeddingsModel` is needed in the pipeline. ## Predicted Entities `ACTIVIDAD`, `PROFESION`, `SITUACION_LABORAL` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_PROFESSIONS_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_PROFESSIONS_ES.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/meddroprof_scielowiki_es_3.1.3_3.0_1627328955264.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/meddroprof_scielowiki_es_3.1.3_3.0_1627328955264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence = SentenceDetector() \ .setInputCols("document") \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols("sentence") \ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d", "es", "clinical/models")\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("meddroprof_scielowiki", "es", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence, tokenizer, word_embeddings, clinical_ner, ner_converter]) sample_text = """La paciente es la mayor de 2 hermanos, tiene un hermano de 13 años estudiando 1o ESO. Sus padres son ambos ATS , trabajan en diferentes centros de salud estudiando 1o ESO""" df = spark.createDataFrame([[sample_text]]).toDF("text") result = pipeline.fit(df).transform(df) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d", "es", "clinical/models") .setInputCols(Array("document", "token")) .setOutputCol("word_embeddings") val clinical_ner = MedicalNerModel.pretrained("meddroprof_scielowiki", "es", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence, tokenizer, word_embeddings, clinical_ner, ner_converter)) val data = Seq("""La paciente es la mayor de 2 hermanos, tiene un hermano de 13 años estudiando 1o ESO. Sus padres son ambos ATS , trabajan en diferentes centros de salud estudiando 1o ESO""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.scielowiki").predict("""La paciente es la mayor de 2 hermanos, tiene un hermano de 13 años estudiando 1o ESO. Sus padres son ambos ATS , trabajan en diferentes centros de salud estudiando 1o ESO""") ```

## Results ```bash +---------------------------------------+-----------------+ |chunk |ner_label | +---------------------------------------+-----------------+ |estudiando 1o ESO |SITUACION_LABORAL| |ATS |PROFESION | |trabajan en diferentes centros de salud|PROFESION | |estudiando 1o ESO |SITUACION_LABORAL| +---------------------------------------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|meddroprof_scielowiki| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Dependencies:|embeddings_scielowiki_300d| ## Data Source The model was trained with the [MEDDOPROF](https://temu.bsc.es/meddoprof/data/) data set: > The MEDDOPROF corpus is a collection of 1844 clinical cases from over 20 different specialties annotated with professions and employment statuses. The corpus was annotated by a team composed of linguists and clinical experts following specially prepared annotation guidelines, after several cycles of quality control and annotation consistency analysis before annotating the entire dataset. Figure 1 shows a screenshot of a sample manual annotation generated using the brat annotation tool. Reference: ``` @article{meddoprof, title={NLP applied to occupational health: MEDDOPROF shared task at IberLEF 2021 on automatic recognition, classification and normalization of professions and occupations from medical texts}, author={Lima-López, Salvador and Farré-Maduell, Eulàlia and Miranda-Escalada, Antonio and Brivá-Iglesias, Vicent and Krallinger, Martin}, journal = {Procesamiento del Lenguaje Natural}, volume = {67}, year={2021} } ``` ## Benchmarking ```bash label precision recall f1-score support B-ACTIVIDAD 0.82 0.36 0.50 25 B-PROFESION 0.87 0.75 0.81 634 B-SITUACION_LABORAL 0.79 0.67 0.72 310 I-ACTIVIDAD 0.86 0.43 0.57 58 I-PROFESION 0.87 0.80 0.83 944 I-SITUACION_LABORAL 0.74 0.71 0.73 407 O 1.00 1.00 1.00 139880 accuracy - - 0.99 142258 macro-avg 0.85 0.67 0.74 142258 weighted-avg 0.99 0.99 0.99 142258 ``` --- layout: model title: Spanish NER Pipeline author: John Snow Labs name: roberta_token_classifier_bne_capitel_ner_pipeline date: 2022-04-20 tags: [roberta, token_classifier, spanish, ner, es, open_source] task: Named Entity Recognition language: es edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_token_classifier_bne_capitel_ner_es](https://nlp.johnsnowlabs.com/2021/12/07/roberta_token_classifier_bne_capitel_ner_es.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bne_capitel_ner_pipeline_es_3.4.1_3.0_1650450203759.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bne_capitel_ner_pipeline_es_3.4.1_3.0_1650450203759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("roberta_token_classifier_bne_capitel_ner_pipeline", lang = "es") pipeline.annotate("Me llamo Antonio y trabajo en la fábrica de Mercedes-Benz en Madrid.") ``` ```scala val pipeline = new PretrainedPipeline("roberta_token_classifier_bne_capitel_ner_pipeline", lang = "es") pipeline.annotate("Me llamo Antonio y trabajo en la fábrica de Mercedes-Benz en Madrid.") ```

## Results ```bash +------------------------+---------+ |chunk |ner_label| +------------------------+---------+ |Antonio |PER | |fábrica de Mercedes-Benz|ORG | |Madrid |LOC | +------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_bne_capitel_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Language:|es| |Size:|459.4 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: Fast Neural Machine Translation Model from Greek Languages to English author: John Snow Labs name: opus_mt_grk_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, grk, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `grk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_grk_en_xx_2.7.0_2.4_1609166792734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_grk_en_xx_2.7.0_2.4_1609166792734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_grk_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Utilities Clause Binary Classifier author: John Snow Labs name: legclf_utilities_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `utilities` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `utilities` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_utilities_clause_en_1.0.0_3.2_1660123178356.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_utilities_clause_en_1.0.0_3.2_1660123178356.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_utilities_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[utilities]| |[other]| |[other]| |[utilities]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_utilities_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.98 0.99 105 utilities 0.93 0.97 0.95 29 accuracy - - 0.98 134 macro-avg 0.96 0.97 0.97 134 weighted-avg 0.98 0.98 0.98 134 ``` --- layout: model title: Word2Vec Embeddings in Pashto (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, ps, open_source] task: Embeddings language: ps edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ps_3.4.1_3.0_1647451184427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ps_3.4.1_3.0_1647451184427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_ontonotes5","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_ontonotes5","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tner_large_ontonotes5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/asahi417/tner-xlm-roberta-large-ontonotes5 - https://github.com/asahi417/tner --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465517 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465517` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465517_en_4.0.0_3.0_1655986325897.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465517_en_4.0.0_3.0_1655986325897.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465517","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465517","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465517.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465517| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465517 --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes author: John Snow Labs name: sbiobertresolve_umls_findings date: 2021-10-03 tags: [entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.2.3 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to 4 major categories of UMLS CUI codes using sbiobert_base_cased_mli Sentence Bert Embeddings. It has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. This model returns CUI (concept unique identifier) codes for 200K concepts from clinical findings.https://www.nlm.nih.gov/research/umls/index.html ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_UMLS_CUI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_3.2.3_3.0_1633220877215.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_3.2.3_3.0_1633220877215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | ner_chunk | cui_code | |---:|:--------------------------------------|:-----------| | 0 | gestational diabetes mellitus | C2183115 | | 1 | subsequent type two diabetes mellitus | C3532488 | | 2 | T2DM | C3280267 | | 3 | HTG-induced pancreatitis | C4554179 | | 4 | an acute hepatitis | C4750596 | | 5 | obesity | C1963185 | | 6 | a body mass index | C0578022 | | 7 | polyuria | C3278312 | | 8 | polydipsia | C3278316 | | 9 | poor appetite | C0541799 | | 10 | vomiting | C0042963 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_findings| |Compatibility:|Healthcare NLP 3.2.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[umls_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on 200K concepts from clinical findings.https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: Legal Withholdings Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_withholdings_bert date: 2023-03-05 tags: [en, legal, classification, clauses, withholdings, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Withholdings` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Withholdings`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_withholdings_bert_en_1.0.0_3.0_1678049980404.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_withholdings_bert_en_1.0.0_3.0_1678049980404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_withholdings_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Withholdings]| |[Other]| |[Other]| |[Withholdings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_withholdings_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.99 0.98 0.98 85 Withholdings 0.97 0.98 0.98 61 accuracy - - 0.98 146 macro-avg 0.98 0.98 0.98 146 weighted-avg 0.98 0.98 0.98 146 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from seongju) author: John Snow Labs name: xlm_roberta_qa_squadv2_xlm_roberta_base date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squadv2-xlm-roberta-base` is a English model originally trained by `seongju`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_squadv2_xlm_roberta_base_en_4.0.0_3.0_1655988029859.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_squadv2_xlm_roberta_base_en_4.0.0_3.0_1655988029859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_squadv2_xlm_roberta_base","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_squadv2_xlm_roberta_base","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_squadv2_xlm_roberta_base| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|875.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/seongju/squadv2-xlm-roberta-base - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: English image_classifier_vit_denver_nyc_paris ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_denver_nyc_paris date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_denver_nyc_paris` is a English model originally trained by nateraw. ## Predicted Entities `denver`, `new york city`, `paris` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_denver_nyc_paris_en_4.1.0_3.0_1660172026182.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_denver_nyc_paris_en_4.1.0_3.0_1660172026182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_denver_nyc_paris", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_denver_nyc_paris", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_denver_nyc_paris| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Embeddings Scielo 50 dims author: John Snow Labs name: embeddings_scielo_50d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-26 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_50d_es_2.5.0_2.4_1590467114993.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_50d_es_2.5.0_2.4_1590467114993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_scielo_50d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_scielo_50d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.scielo.50d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_scielo_50d``. {:.model-param} ## Model Information {:.table-model} |---------------|-----------------------| | Name: | embeddings_scielo_50d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | es | | Dimension: | 50.0 | {:.h2_title} ## Data Source Trained on Scielo Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from IIC) author: John Snow Labs name: roberta_qa_base_spanish_squades date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades` is a Spanish model originally trained by `IIC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_es_4.2.4_3.0_1669986476235.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_es_4.2.4_3.0_1669986476235.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|459.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/IIC/roberta-base-spanish-squades - https://arxiv.org/abs/2107.07253 - https://paperswithcode.com/sota?task=question-answering&dataset=squad_es --- layout: model title: Detect Clinical Conditions (ner_eu_clinical_condition - es) author: John Snow Labs name: ner_eu_clinical_condition date: 2023-02-06 tags: [es, clinical, licensed, ner, clinical_condition] task: Named Entity Recognition language: es edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition (NER) deep learning model for extracting clinical conditions from Spanish texts. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nichols, Named Entity Recognition with Bidirectional LSTM-CNN. The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Predicted Entities `clinical_condition` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_es_4.2.8_3.0_1675721390266.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_es_4.2.8_3.0_1675721390266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","es")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_eu_clinical_condition', "es", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""La exploración abdominal revela una cicatriz de laparotomía media infraumbilical, la presencia de ruidos disminuidos, y dolor a la palpación de manera difusa sin claros signos de irritación peritoneal. No existen hernias inguinales o crurales."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","es") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_eu_clinical_condition", "es", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""La exploración abdominal revela una cicatriz de laparotomía media infraumbilical, la presencia de ruidos disminuidos, y dolor a la palpación de manera difusa sin claros signos de irritación peritoneal. No existen hernias inguinales o crurales.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +--------------------+------------------+ |chunk |ner_label | +--------------------+------------------+ |cicatriz |clinical_condition| |dolor a la palpación|clinical_condition| |signos |clinical_condition| |irritación |clinical_condition| |hernias inguinales |clinical_condition| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_condition| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|898.1 KB| ## References The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Benchmarking ```bash label tp fp fn total precision recall f1 clinical_condition 354.0 42.0 84.0 438.0 0.8939 0.8082 0.8489 macro - - - - - - 0.8489 micro - - - - - - 0.8489 ``` --- layout: model title: NER Model Finder with Sentence Entity Resolvers (SBert, Medium, Uncased) author: John Snow Labs name: sbertresolve_ner_model_finder date: 2022-01-17 tags: [ner, licensed, clinical, entity_resolver, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.2 spark_version: 2.4 supported: true recommended: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities (NER labels) to the most appropriate NER model using `sbert_jsl_medium_uncased` Sentence Bert Embeddings. Given the entity name, it will return a list of pretrained NER models having that entity or similar ones. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_ner_model_finder_en_3.3.2_2.4_1642422477025.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_ner_model_finder_en_3.3.2_2.4_1642422477025.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_uncased","en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") ner_model_finder = SentenceEntityResolverModel\ .pretrained("sbertresolve_ner_model_finder", "en", "clinical/models")\ .setInputCols(["ner_chunk", "sbert_embeddings"])\ .setOutputCol("model_names")\ .setDistanceFunction("EUCLIDEAN") ner_model_finder_pipelineModel = PipelineModel(stages = [documentAssembler, sbert_embedder, ner_model_finder]) light_pipeline = LightPipeline(ner_model_finder_pipelineModel) annotations = light_pipeline.fullAnnotate("medication") ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val ner_model_finder = SentenceEntityResolverModel .pretrained("sbertresolve_ner_model_finder", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("model_names") .setDistanceFunction("EUCLIDEAN") val ner_model_finder_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, ner_model_finder)) val light_pipeline = LightPipeline(ner_model_finder_pipelineModel) val annotations = light_pipeline.fullAnnotate("medication") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.ner.model_finder").predict("""Put your text here.""") ```

## Results ```bash +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |entity |models |all_models |resolutions | +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |medication|['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_ade_clinical', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']|['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_ade_clinical', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']:::['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_clinical_large', 'ner_healthcare', 'ner_jsl_enriched', 'ner_clinical', 'ner_jsl_slim', 'ner_covid_trials', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_events_admission_clinical', 'ner_events_healthcare', 'ner_events_clinical', 'ner_jsl_greedy']:::['ner_medmentions_coarse']:::['ner_jsl_enriched', 'ner_covid_trials', 'ner_jsl', 'ner_medmentions_coarse']:::['ner_drugs']:::['ner_clinical_icdem', 'ner_medmentions_coarse']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_medmentions_coarse', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_medmentions_coarse', 'ner_radiology_wip_clinical', 'ner_jsl_slim', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy', 'ner_radiology']:::['ner_medmentions_coarse','ner_clinical_icdem']:::['ner_posology_experimental']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_measurements_clinical', 'ner_radiology_wip_clinical', 'ner_radiology']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_posology_small', 'ner_jsl_enriched', 'ner_posology_experimental', 'ner_posology_large', 'ner_posology_healthcare', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_posology_greedy', 'ner_posology', 'ner_jsl_greedy']:::['ner_covid_trials', 'ner_medmentions_coarse', 'jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_deid_subentity_augmented', 'ner_deid_subentity_glove', 'ner_deidentify_dl', 'ner_deid_enriched']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_covid_trials', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_medmentions_coarse', 'jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_chemd_clinical']|medication:::drug:::treatment:::therapeutic procedure:::drug ingredient:::drug chemical:::diagnostic aid:::substance:::medical device:::diagnostic procedure:::administration:::measurement:::drug strength:::physiological reaction:::patient:::vaccine:::psychological condition:::abbreviation| +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_ner_model_finder| |Compatibility:|Healthcare NLP 3.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sbert_embeddings]| |Output Labels:|[models]| |Language:|en| |Size:|611.1 KB| |Case sensitive:|false| --- layout: model title: Fast Neural Machine Translation Model from Afrikaans to Swedish author: John Snow Labs name: opus_mt_af_sv date: 2021-06-01 tags: [open_source, seq2seq, translation, af, sv, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: af target languages: sv {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_af_sv_xx_3.1.0_2.4_1622562929786.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_af_sv_xx_3.1.0_2.4_1622562929786.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_af_sv| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune TFWav2Vec2ForCTC from hrdipto author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune` is a English model originally trained by hrdipto. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune_en_4.2.0_3.0_1664041738598.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune_en_4.2.0_3.0_1664041738598.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_bangla_command_generated_data_finetune| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate English to Dutch Pipeline author: John Snow Labs name: translate_en_nl date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, nl, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `nl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_nl_xx_2.7.0_2.4_1609688238018.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_nl_xx_2.7.0_2.4_1609688238018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_nl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_nl", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.nl').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_nl| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Named Entity Recognition - BERT Base (OntoNotes) author: John Snow Labs name: onto_bert_base_cased date: 2020-12-05 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ner, en, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Onto is a Named Entity Recognition (or NER) model trained on OntoNotes 5.0. It can extract up to 18 entities such as people, places, organizations, money, time, date, etc. This model uses the pretrained `bert_base_cased` embeddings model from `BertEmbeddings` annotator as an input. ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_bert_base_cased_en_2.7.0_2.4_1607197077494.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_bert_base_cased_en_2.7.0_2.4_1607197077494.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ner_onto = NerDLModel.pretrained("onto_bert_base_cased", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_base_cased", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_onto = NerDLModel.pretrained("onto_bert_base_cased", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.onto.bert.cased_base').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PERSON | |October 28, 1955 |DATE | |American |NORP | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PERSON | |May 2014 |DATE | |one |CARDINAL | |the 1970s and 1980s |DATE | |Seattle |GPE | |Washington |GPE | |Gates |PERSON | |Paul Allen |PERSON | |1975 |DATE | |Albuquerque |GPE | |New Mexico |GPE | |Gates |ORG | |January 2000 |DATE | |the late 1990s |DATE | |Gates |PERSON | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_bert_base_cased| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source The model is trained based on data from [OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) ## Benchmarking ```bash Micro-average: prec: 0.8987879, rec: 0.90063596, f1: 0.89971095 CoNLL Eval: processed 152728 tokens with 11257 phrases; found: 11276 phrases; correct: 10006. accuracy: 98.01%; 10006 11257 11276 precision: 88.74%; recall: 88.89%; FB1: 88.81 CARDINAL: 822 935 990 precision: 83.03%; recall: 87.91%; FB1: 85.40 990 DATE: 1355 1602 1567 precision: 86.47%; recall: 84.58%; FB1: 85.52 1567 EVENT: 32 63 59 precision: 54.24%; recall: 50.79%; FB1: 52.46 59 FAC: 96 135 124 precision: 77.42%; recall: 71.11%; FB1: 74.13 124 GPE: 2116 2240 2182 precision: 96.98%; recall: 94.46%; FB1: 95.70 2182 LANGUAGE: 10 22 11 precision: 90.91%; recall: 45.45%; FB1: 60.61 11 LAW: 21 40 28 precision: 75.00%; recall: 52.50%; FB1: 61.76 28 LOC: 141 179 178 precision: 79.21%; recall: 78.77%; FB1: 78.99 178 MONEY: 278 314 321 precision: 86.60%; recall: 88.54%; FB1: 87.56 321 NORP: 799 841 850 precision: 94.00%; recall: 95.01%; FB1: 94.50 850 ORDINAL: 177 195 217 precision: 81.57%; recall: 90.77%; FB1: 85.92 217 ORG: 1606 1795 1848 precision: 86.90%; recall: 89.47%; FB1: 88.17 1848 PERCENT: 306 349 344 precision: 88.95%; recall: 87.68%; FB1: 88.31 344 PERSON: 1856 1988 1978 precision: 93.83%; recall: 93.36%; FB1: 93.60 1978 PRODUCT: 54 76 76 precision: 71.05%; recall: 71.05%; FB1: 71.05 76 QUANTITY: 87 105 108 precision: 80.56%; recall: 82.86%; FB1: 81.69 108 TIME: 143 212 216 precision: 66.20%; recall: 67.45%; FB1: 66.82 216 WORK_OF_ART: 107 166 179 precision: 59.78%; recall: 64.46%; FB1: 62.03 179 ``` --- layout: model title: Traditional Chinese Word Segmentation author: John Snow Labs name: wordseg_gsd_ud_trad date: 2021-01-25 task: Word Segmentation language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [word_segmentation, zh, open_source] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know a priori which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. This model was trained on *traditional characters* in Chinese texts. Reference: - Xue, Nianwen. “Chinese word segmentation as character tagging.” International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_trad_zh_2.7.0_2.4_1611584735643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_trad_zh_2.7.0_2.4_1611584735643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline as a substitute for the Tokenizer stage.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud_trad", "zh")\ .setInputCols(["sentence"])\ .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler,sentence_detector, word_segmenter]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['然而，這樣的處理也衍生了一些問題。']], ["text"]) result = ws_model.transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud_trad", "zh") .setInputCols(Array("sentence")) .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler,sentence_detector, word_segmenter)) val data = Seq("然而，這樣的處理也衍生了一些問題。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，這樣的處理也衍生了一些問題。"""] token_df = nlu.load('zh.segment_words.gsd').predict(text) token_df ```

## Results ```bash +-----------------------------------------+-----------------------------------------------------------+ |text | result | +-----------------------------------------+-----------------------------------------------------------+ |然而，這樣的處理也衍生了一些問題。 |[然而, ，, 這樣, 的, 處理, 也, 衍生, 了, 一些, 問題, 。] | +-----------------------------------------+-----------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_gsd_ud_trad| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|zh| ## Data Source The model was trained on the [Universal Dependencies](https://universaldependencies.org/) for Traditional Chinese annotated and converted by Google. ## Benchmarking ```bash | precision | recall | f1-score | |--------------|----------|------------| | 0.7392 | 0.7754 | 0.7569 | ``` --- layout: model title: Legal Distributions Clause Binary Classifier author: John Snow Labs name: legclf_distributions_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `distributions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `distributions` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_distributions_clause_en_1.0.0_3.2_1660123426386.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_distributions_clause_en_1.0.0_3.2_1660123426386.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_distributions_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[distributions]| |[other]| |[other]| |[distributions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_distributions_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support distributions 0.88 0.90 0.89 41 other 0.93 0.92 0.92 60 accuracy - - 0.91 101 macro-avg 0.91 0.91 0.91 101 weighted-avg 0.91 0.91 0.91 101 ``` --- layout: model title: Legal Master Administrative Services Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_master_administrative_services_agreement_bert date: 2023-01-29 tags: [en, legal, classification, master, administrative, services, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_master_administrative_services_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `master-administrative-services-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `master-administrative-services-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_master_administrative_services_agreement_bert_en_1.0.0_3.0_1674990113596.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_master_administrative_services_agreement_bert_en_1.0.0_3.0_1674990113596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_master_administrative_services_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[master-administrative-services-agreement]| |[other]| |[other]| |[master-administrative-services-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_master_administrative_services_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support master-administrative-services-agreement 1.00 1.00 1.00 54 other 1.00 1.00 1.00 101 accuracy - - 1.00 155 macro-avg 1.00 1.00 1.00 155 weighted-avg 1.00 1.00 1.00 155 ``` --- layout: model title: Clinical Deidentification (English, Glove, Augmented) author: John Snow Labs name: clinical_deidentification_glove_augmented date: 2022-03-22 tags: [deid, deidentification, en, licensed] task: De-identification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true recommended: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with lightweight glove_100d embeddings and can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR` entities. It's different to `clinical_deidentification_glove` in the way it manages PHONE and PATIENT, having apart from the NER, some rules in Contextual Parser components. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_3.4.1_3.0_1647966639326.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_3.4.1_3.0_1647966639326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification_glove", "en", "clinical/models") deid_pipeline.annotate("Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN:324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years-old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = PretrainedPipeline("clinical_deidentification_glove","en","clinical/models") val result = pipeline.annotate("Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN:324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years-old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.glove_augmented.pipeline").predict("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN:324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years-old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ```

## Results ```bash {'sentence': ['Record date : 2093-01-13, David Hale, M.D.', 'IP: 203.120.223.13.', 'The driver's license no:A334455B.', 'the SSN:324598674 and e-mail: hale@gmail.com.', 'Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93.', 'PCP : Oliveira, 25 years-old.', 'Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.'], 'masked': ['Record date : , , M.D.', 'IP: .', 'The driver's license .', 'the and e-mail: .', 'Name : MR. # Date : .', 'PCP : , years-old.', 'Record date : , Patient's VIN : .'], 'obfuscated': ['Record date : 2093-02-13, Shella Solan, M.D.', 'IP: 444.444.444.444.', 'The driver's license O497302436569.', 'the SSN-539-29-1060 and e-mail: Keith@google.com.', 'Name : Roscoe Kerns MR. # Q984288 Date : 10-08-1991.', 'PCP : Dr Rudell Dubin, 10 years-old.', 'Record date : 2079-12-30, Patient's VIN : 5eeee44ffff555666.'], 'ner_chunk': ['2093-01-13', 'David Hale', 'no:A334455B', 'SSN:324598674', 'Hendrickson, Ora', '719435', '01/13/93', 'Oliveira', '25', '2079-11-09', '1HGBH41JXMN109286']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification_glove_augmented| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|181.3 MB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetector - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - medical.NerConverterInternal - medical.NerModel - medical.NerConverterInternal - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squadv1-x1.16-f88.1-d8-unstruct-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1_en_4.0.0_3.0_1654181545377.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1_en_4.0.0_3.0_1654181545377.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.x1.16_f88.1_d8_unstruct.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squadv1_x1.16_f88.1_d8_unstruct_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|146.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squadv1-x1.16-f88.1-d8-unstruct-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el8_dl2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el8-dl2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_dl2_en_4.3.0_3.0_1675120612912.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_dl2_en_4.3.0_3.0_1675120612912.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el8_dl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el8_dl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el8_dl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|127.7 MB| ## References - https://huggingface.co/google/t5-efficient-small-el8-dl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Cased model (from ybagoury) author: John Snow Labs name: t5_flan_base_tldr_news date: 2023-03-02 tags: [open_source, t5, flan, en, tensorflow] task: Text Generation language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. flan-t5-base-tldr_news is a English model originally trained by ybagoury. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_flan_base_tldr_news_en_4.3.0_3.0_1677760144575.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_flan_base_tldr_news_en_4.3.0_3.0_1677760144575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_flan_base_tldr_news","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_flan_base_tldr_news","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_flan_base_tldr_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|1.0 GB| ## References https://huggingface.co/ybagoury/flan-t5-base-tldr_news --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673_de_4.2.0_3.0_1664114173921.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673_de_4.2.0_3.0_1664114173921.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s673| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Part of Speech for Amharic (pos_ud_att) author: John Snow Labs name: pos_ud_att date: 2021-01-20 task: Part of Speech Tagging language: am edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [am, pos, open_source] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 13 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. ## Predicted Entities | POS tag | Description | |---------|----------------------------| | ADJ | adjective | | ADP | adposition | | ADV | adverb | | AUX | auxiliary | | CCONJ | coordinating conjunction | | DET | determiner | | INTJ | interjection | | NOUN | noun | | NUM | numeral | | PART | particle | | PRON | pronoun | | PROPN | proper noun | | PUNCT | punctuation | | SCONJ | subordinating conjunction | | SYM | symbol | | VERB | verb | | X | other | {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_att_am_2.7.0_2.4_1611180723328.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_att_am_2.7.0_2.4_1611180723328.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_ud_att", "am") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['ልጅ ኡ ን ሥራ ው ን አስጨርስ ኧው ኣል ኧሁ ።']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_ud_att", "am") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("ልጅ ኡ ን ሥራ ው ን አስጨርስ ኧው ኣል ኧሁ ።").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["ልጅ ኡ ን ሥራ ው ን አስጨርስ ኧው ኣል ኧሁ ።"] pos_df = nlu.load('am.pos').predict(text) pos_df ```

## Results ```bash +------------------------------+----------------------------------------------------------------+ |text |result | +------------------------------+----------------------------------------------------------------+ |ልጅ ኡ ን ሥራ ው ን አስጨርስ ኧው ኣል ኧሁ ።|[NOUN, DET, PART, NOUN, DET, PART, VERB, PRON, AUX, PRON, PUNCT]| +------------------------------+----------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_att| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|am| ## Data Source The model was trained on the [Universal Dependencies](https://universaldependencies.org/) version 2.7. Reference: - Binyam Ephrem Seyoum ,Yusuke Miyao and Baye Yimam Mekonnen.2018.Universal Dependencies for Amharic. In Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018), pp. 2216–2222, Miyazaki, Japan: European Language Resources Association (ELRA) ## Benchmarking ```bash | | precision | recall | f1-score | support | |:------------:|:---------:|:------:|:--------:|:-------:| | ADJ | 1.00 | 0.97 | 0.99 | 116 | | ADP | 0.99 | 1.00 | 0.99 | 681 | | ADV | 0.94 | 0.99 | 0.96 | 93 | | AUX | 1.00 | 1.00 | 1.00 | 419 | | CCONJ | 0.99 | 0.97 | 0.98 | 99 | | DET | 0.99 | 1.00 | 0.99 | 485 | | INTJ | 0.97 | 0.99 | 0.98 | 67 | | NOUN | 0.99 | 1.00 | 1.00 | 1485 | | NUM | 1.00 | 1.00 | 1.00 | 42 | | PART | 1.00 | 1.00 | 1.00 | 875 | | PRON | 1.00 | 1.00 | 1.00 | 2547 | | PROPN | 1.00 | 0.99 | 0.99 | 236 | | PUNCT | 1.00 | 1.00 | 1.00 | 1093 | | SCONJ | 1.00 | 0.98 | 0.99 | 214 | | VERB | 1.00 | 1.00 | 1.00 | 1552 | | accuracy | | | 1.00 | 10004 | | macro avg | 0.99 | 0.99 | 0.99 | 10004 | | weighted avg | 1.00 | 1.00 | 1.00 | 10004 | ``` --- layout: model title: Legal Approvals Clause Binary Classifier author: John Snow Labs name: legclf_approvals_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `approvals` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `approvals` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_approvals_clause_en_1.0.0_3.2_1660123231676.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_approvals_clause_en_1.0.0_3.2_1660123231676.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_approvals_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[approvals]| |[other]| |[other]| |[approvals]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_approvals_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support approvals 0.89 0.94 0.91 33 other 0.97 0.95 0.96 82 accuracy - - 0.95 115 macro-avg 0.93 0.95 0.94 115 weighted-avg 0.95 0.95 0.95 115 ``` --- layout: model title: Ocr pipeline in streaming author: John Snow Labs name: ocr_streaming date: 2023-01-03 tags: [en, licensed, ocr, streaming] task: Ocr Streaming language: en nav_key: models edition: Visual NLP 4.0.0 spark_version: 3.0 supported: true annotator: OcrStreaming article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Streaming pipeline implementation for the OCR task, using tesseract models. Tesseract is an Optical Character Recognition (OCR) engine developed by Google. It is an open-source tool that can be used to recognize text in images and convert it into machine-readable text. The engine is based on a neural network architecture and uses machine learning algorithms to improve its accuracy over time. Tesseract has been trained on a variety of datasets to improve its recognition capabilities. These datasets include images of text in various languages and scripts, as well as images with different font styles, sizes, and orientations. The training process involves feeding the engine with a large number of images and their corresponding text, allowing the engine to learn the patterns and characteristics of different text styles. One of the most important datasets used in training Tesseract is the UNLV dataset, which contains over 400,000 images of text in different languages, scripts, and font styles. This dataset is widely used in the OCR community and has been instrumental in improving the accuracy of Tesseract. Other datasets that have been used in training Tesseract include the ICDAR dataset, the IIIT-HWS dataset, and the RRC-GV-WS dataset. In addition to these datasets, Tesseract also uses a technique called adaptive training, where the engine can continuously improve its recognition capabilities by learning from new images and text. This allows Tesseract to adapt to new text styles and languages, and improve its overall accuracy. ## Predicted Entities {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/blob/master/tutorials/Certification_Trainings/6.1.SparkOcrStreamingPDF.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python # Transform binary to image pdf_to_image = PdfToImage() \ .setInputCol("content") \ .setOutputCol("image") # Run OCR for each region ocr = ImageToText() \ .setInputCol("image") \ .setOutputCol("text") \ .setConfidenceThreshold(60) # OCR pipeline pipeline = PipelineModel(stages=[ pdf_to_image, ocr]) # fill path to folder with PDF's here dataset_path = "data/pdfs/*.pdf" # read one file for infer schema pdfs_df = spark.read.format("binaryFile").load(dataset_path).limit(1) # count of files in one microbatch maxFilesPerTrigger = 4 # read files as stream pdf_stream_df = spark.readStream \ .format("binaryFile") \ .schema(pdfs_df.schema) \ .option("maxFilesPerTrigger", maxFilesPerTrigger) \ .load(dataset_path) # process files using OCR pipeline result = pipeline.transform(pdf_stream_df).withColumn("timestamp", current_timestamp()) # store results to memory table query = result.writeStream \ .format('memory') \ .queryName('result') \ .start() # show results spark.table("result").select("timestamp","pagenum", "path", "text").show(10) ``` ```scala # Transform binary to image val pdf_to_image = new PdfToImage() .setInputCol("content") .setOutputCol("image") # Run OCR for each region val ocr = new ImageToText() .setInputCol("image") .setOutputCol("text") .setConfidenceThreshold(60) # OCR pipeline val pipeline = new PipelineModel().setStages(Array( pdf_to_image, ocr)) # fill path to folder with PDF's here val dataset_path = "data/pdfs/*.pdf" # read one file for infer schema val pdfs_df = spark.read.format("binaryFile").load(dataset_path).limit(1) # count of files in one microbatch val maxFilesPerTrigger = 4 # read files as stream val pdf_stream_df = spark.readStream .format("binaryFile") .schema(pdfs_df.schema) .option("maxFilesPerTrigger", maxFilesPerTrigger) .load(dataset_path) # process files using OCR pipeline val result = pipeline.transform(pdf_stream_df).withColumn("timestamp", current_timestamp()) # store results to memory table val query = result.writeStream .format("memory") .queryName("result") .start() # show results spark.table("result").select(Array("timestamp","pagenum", "path", "text")).show(10) ```

## Example ### Input: ![Screenshot](/assets/images/examples_ocr/image4.png) ### Output ```bash +--------------------+ | value| +--------------------+ | | | | | | | | | | | | |ne Pa a Date: 7/1...| |er ‘Sample No. _ ...| |“ Original reques...| | | |Sample specificat...| | , BLEND CASING R...| | | |- OLD GOLD STRAIG...| | | |Control for Sampl...| | | | Cigarettes:| | | | OLD GOLD STRAIGHT| +--------------------+ only showing top 20 rows ``` ## Model Information {:.table-model} |---|---| |Model Name:|ocr_streaming| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Legal Indemnification Procedures Clause Binary Classifier author: John Snow Labs name: legclf_indemnification_procedures_clause date: 2023-01-27 tags: [en, legal, classification, indemnification, procedures, clauses, indemnification_procedures, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `indemnification-procedures` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `indemnification-procedures`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_procedures_clause_en_1.0.0_3.0_1674819938050.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_procedures_clause_en_1.0.0_3.0_1674819938050.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_indemnification_procedures_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[indemnification-procedures]| |[other]| |[other]| |[indemnification-procedures]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnification_procedures_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support indemnification-procedures 1.00 0.96 0.98 23 other 0.97 1.00 0.99 39 accuracy - - 0.98 62 macro-avg 0.99 0.98 0.98 62 weighted-avg 0.98 0.98 0.98 62 ``` --- layout: model title: Detect Problems, Tests and Treatments (ner_clinical) in German author: John Snow Labs name: ner_clinical date: 2023-05-08 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terms in German. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `PROBLEM`, `TEST`, `TREATMENT` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/14.German_Healthcare_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_de_4.4.0_3.0_1683555292486.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_de_4.4.0_3.0_1683555292486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "de", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) sample_text= """Verschlechterung von Schmerzen oder Schwäche in den Beinen , Verlust der Darm - oder Blasenfunktion oder andere besorgniserregende Symptome. Der Patient erhielt empirisch Ampicillin , Gentamycin und Flagyl sowie Narcan zur Umkehrung von Fentanyl . ALT war 181 , AST war 156 , LDH war 336 , alkalische Phosphatase war 214 und Bilirubin war insgesamt 12,7 .""" results = model.transform(spark.createDataFrame([[sample_text]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "de", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_clinical", "de", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""Verschlechterung von Schmerzen oder Schwäche in den Beinen , Verlust der Darm - oder Blasenfunktion oder andere besorgniserregende Symptome. Der Patient erhielt empirisch Ampicillin , Gentamycin und Flagyl sowie Narcan zur Umkehrung von Fentanyl . ALT war 181 , AST war 156 , LDH war 336 , alkalische Phosphatase war 214 und Bilirubin war insgesamt 12,7 .""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Schmerzen |PROBLEM | |Schwäche in den Beinen|PROBLEM | |Verlust der Darm |PROBLEM | |Blasenfunktion |PROBLEM | |Symptome |PROBLEM | |empirisch Ampicillin |TREATMENT| |Gentamycin |TREATMENT| |Flagyl |TREATMENT| |Narcan |TREATMENT| |Fentanyl |TREATMENT| |ALT |TEST | |AST |TEST | |LDH |TEST | |alkalische Phosphatase|TEST | |Bilirubin |TEST | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| |Size:|2.0 MB| ## Benchmarking ```bash label precision recall f1-score support B-PROBLEM 0.85 0.71 0.78 512 B-TEST 0.89 0.85 0.87 203 B-TREATMENT 0.84 0.82 0.83 238 I-PROBLEM 0.78 0.70 0.74 355 I-TEST 0.90 0.83 0.87 66 I-TREATMENT 0.62 0.71 0.66 75 O 0.94 0.97 0.95 4141 accuracy - - 0.91 5590 macro avg 0.83 0.80 0.81 5590 weighted avg 0.91 0.91 0.91 5590 ``` --- layout: model title: English DistilBertForTokenClassification Cased model (from f2io) author: John Snow Labs name: distilbert_token_classifier_ner_roles_openapi date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ner-roles-openapi` is a English model originally trained by `f2io`. ## Predicted Entities ``, `MISC`, `ORG`, `ENTITY`, `PER`, `PRG`, `ROLE`, `OR`, `LOC`, `ACTION` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.1_3.0_1678782949346.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.1_3.0_1678782949346.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_ner_roles_openapi| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/f2io/ner-roles-openapi --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from deepset) author: John Snow Labs name: roberta_qa_deepset_base_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true recommended: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_en_4.2.4_3.0_1669986722225.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_en_4.2.4_3.0_1669986722225.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_base_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/deepset/roberta-base-squad2 - https://haystack.deepset.ai/tutorials/first-qa-system - https://github.com/deepset-ai/haystack/ - https://haystack.deepset.ai/tutorials/first-qa-system - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - http://deepset.ai/ - https://haystack.deepset.ai/ - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/haystack - https://docs.haystack.deepset.ai - https://haystack.deepset.ai/community - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs - https://paperswithcode.com/sota?task=Question+Answering&dataset=squad_v2 --- layout: model title: Fast Neural Machine Translation Model from Catalan to English author: John Snow Labs name: opus_mt_ca_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ca, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ca` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ca_en_xx_2.7.0_2.4_1609169355424.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ca_en_xx_2.7.0_2.4_1609169355424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ca_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect PHI for Deidentification (Glove, 7 labels) author: John Snow Labs name: ner_deid_generic_glove_pipeline date: 2023-03-13 tags: [deid, clinical, glove, licensed, ner, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_generic_glove](https://nlp.johnsnowlabs.com/2021/06/06/ner_deid_generic_glove_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_glove_pipeline_en_4.3.0_3.2_1678734514341.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_glove_pipeline_en_4.3.0_3.2_1678734514341.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_generic_glove_pipeline", "en", "clinical/models") text = '''Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_generic_glove_pipeline", "en", "clinical/models") val text = "Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:------------|-------------:| | 0 | 2093-01-13 | 14 | 23 | DATE | 1 | | 1 | David Hale | 27 | 36 | NAME | 0.9938 | | 2 | Hendrickson Ora | 55 | 69 | NAME | 0.992 | | 3 | 7194334 | 78 | 84 | ID | 1 | | 4 | 01/13/93 | 93 | 100 | DATE | 1 | | 5 | Oliveira | 110 | 117 | NAME | 1 | | 6 | 25 | 121 | 122 | AGE | 0.8724 | | 7 | 2079-11-09 | 150 | 159 | DATE | 1 | | 8 | Cocke County Baptist Hospital | 163 | 191 | LOCATION | 0.8586 | | 9 | 0295 Keats Street | 195 | 211 | LOCATION | 0.948667 | | 10 | 302-786-5227 | 221 | 232 | CONTACT | 0.9972 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_glove_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|167.3 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Word2Vec Embeddings in Upper Sorbian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, hsb, open_source] task: Embeddings language: hsb edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_hsb_3.4.1_3.0_1647465128124.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_hsb_3.4.1_3.0_1647465128124.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_parsbert_ner_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/HooshvareLab/bert-base-parsbert-ner-uncased - https://arxiv.org/abs/2005.12515 - http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/ - https://github.com/HaniehP/PersianNER - https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://arxiv.org/abs/2005.12515 - https://tensorflow.org/tfrc - https://hooshvare.com - https://www.linkedin.com/in/m3hrdadfi/ - https://twitter.com/m3hrdadfi - https://github.com/m3hrdadfi - https://www.linkedin.com/in/mohammad-gharachorloo/ - https://twitter.com/MGharachorloo - https://github.com/baarsaam - https://www.linkedin.com/in/marziehphi/ - https://twitter.com/marziehphi - https://github.com/marziehphi - https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/ - https://twitter.com/mmanthouri - https://github.com/mmanthouri - https://hooshvare.com/ - https://www.linkedin.com/company/hooshvare - https://twitter.com/hooshvare - https://github.com/hooshvare - https://www.instagram.com/hooshvare/ - https://www.linkedin.com/in/sara-tabrizi-64548b79/ - https://www.behance.net/saratabrizi - https://www.instagram.com/sara_b_tabrizi/ --- layout: model title: Legal Waiver Of Jury Trials Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_waiver_of_jury_trials_bert date: 2023-03-05 tags: [en, legal, classification, clauses, waiver_of_jury_trials, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Waiver_Of_Jury_Trials` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Waiver_Of_Jury_Trials`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_waiver_of_jury_trials_bert_en_1.0.0_3.0_1678050634484.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_waiver_of_jury_trials_bert_en_1.0.0_3.0_1678050634484.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_waiver_of_jury_trials_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Waiver_Of_Jury_Trials]| |[Other]| |[Other]| |[Waiver_Of_Jury_Trials]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_waiver_of_jury_trials_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.85 0.99 0.92 125 Waiver_Of_Jury_Trials 0.99 0.75 0.85 89 accuracy - - 0.89 214 macro-avg 0.92 0.87 0.88 214 weighted-avg 0.91 0.89 0.89 214 ``` --- layout: model title: Mapping RxNorm Codes with Corresponding Actions and Treatments author: John Snow Labs name: rxnorm_action_treatment_mapper date: 2022-05-08 tags: [en, chunk_mapper, rxnorm, action, treatment, licensed, clinical] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps RxNorm and RxNorm Extension codes with their corresponding action and treatment. Action refers to the function of the drug in various body systems; treatment refers to which disease the drug is used to treat. ## Predicted Entities `action`, `treatment` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_action_treatment_mapper_en_3.5.1_3.0_1652043181565.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_action_treatment_mapper_en_3.5.1_3.0_1652043181565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('ner_chunk') sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sentence_embeddings")\ .setCaseSensitive(False) rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") \ .setInputCols(["sentence_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper_action = ChunkMapperModel.pretrained("rxnorm_action_treatment_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("Action")\ .setRel("Action") chunkerMapper_treatment = ChunkMapperModel.pretrained("rxnorm_action_treatment_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("Treatment")\ .setRel("Treatment") pipeline = Pipeline().setStages([document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_action, chunkerMapper_treatment ]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) light_pipeline = LightPipeline(model) result = light_pipeline.annotate(['Sinequan 150 MG', 'Zonalon 50 mg']) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sentence_embeddings") .setCaseSensitive(False) val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") .setInputCols(Array("sentence_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper_action = ChunkMapperModel.pretrained("rxnorm_action_treatment_mapper", "en", "clinical/models")) .setInputCols("rxnorm_code") .setOutputCol("Action") .setRel("Action") val chunkerMapper_treatment = ChunkMapperModel.pretrained("rxnorm_action_treatment_mapper", "en", "clinical/models")) .setInputCols("rxnorm_code") .setOutputCol("Treatment") .setRel("Treatment") val pipeline = new Pipeline().setStages(Array(document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_action, chunkerMapper_treatment )) val text_data = Seq("Sinequan 150 MG", "Zonalon 50 mg").toDS.toDF("text") val res = pipeline.fit(text_data).transform(text_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.rxnorm_to_action_treatment").predict("""Sinequan 150 MG""") ```

## Results ```bash | | ner_chunk | rxnorm_code | Treatment | Action | |---:|:--------------------|:--------------|:-------------------------------------------------------------------------------|:-----------------------------------------------------------------------| | 0 | ['Sinequan 150 MG'] | ['1000067'] | ['Alcoholism', 'Depression', 'Neurosis', 'Anxiety&Panic Attacks', 'Psychosis'] | ['Antidepressant', 'Anxiolytic', 'Psychoanaleptics', 'Sedative'] | | 1 | ['Zonalon 50 mg'] | ['103971'] | ['Pain'] | ['Analgesic', 'Analgesic (Opioid)', 'Analgetic', 'Opioid', 'Vitamins'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_action_treatment_mapper| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|19.3 MB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `recipe_triplet_recipes-roberta-base_TIMESTEP_squadv2_epochs_3` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3_en_4.3.0_3.0_1674212220798.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3_en_4.3.0_3.0_1674212220798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_recipe_triplet_recipes_base_timestep_squadv2_epochs_3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/recipe_triplet_recipes-roberta-base_TIMESTEP_squadv2_epochs_3 --- layout: model title: Legal Rules of construction Clause Binary Classifier author: John Snow Labs name: legclf_rules_of_construction_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `rules-of-construction` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `rules-of-construction` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_rules_of_construction_clause_en_1.0.0_3.2_1660122976182.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_rules_of_construction_clause_en_1.0.0_3.2_1660122976182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_rules_of_construction_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[rules-of-construction]| |[other]| |[other]| |[rules-of-construction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_rules_of_construction_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.99 0.97 98 rules-of-construction 0.98 0.91 0.94 46 accuracy - - 0.97 144 macro-avg 0.97 0.95 0.96 144 weighted-avg 0.97 0.97 0.96 144 ``` --- layout: model title: Translate Dravidian languages to English Pipeline author: John Snow Labs name: translate_dra_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, dra, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `dra` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_dra_en_xx_2.7.0_2.4_1609686505330.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_dra_en_xx_2.7.0_2.4_1609686505330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_dra_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_dra_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.dra.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_dra_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Compliance Clause Binary Classifier author: John Snow Labs name: legclf_compliance_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `compliance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `compliance` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_compliance_clause_en_1.0.0_3.2_1660122240327.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_compliance_clause_en_1.0.0_3.2_1660122240327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_compliance_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[compliance]| |[other]| |[other]| |[compliance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_compliance_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support compliance 0.80 0.82 0.81 67 other 0.94 0.93 0.93 188 accuracy - - 0.90 255 macro-avg 0.87 0.87 0.87 255 weighted-avg 0.90 0.90 0.90 255 ``` --- layout: model title: Sentence Embeddings - sbiobert (tuned) author: John Snow Labs name: sbiobert_jsl_umls_cased date: 2021-06-30 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_umls_cased_en_3.1.0_2.4_1625050246280.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_umls_cased_en_3.1.0_2.4_1625050246280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_jsl_umls_cased","en","clinical/models").setInputCols(["sentence"]).setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbiobert_jsl_umls_cased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.biobert.jsl_umls_cased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobert_jsl_umls_cased| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Tuned on MedNLI dataset ## Benchmarking ```bash MedNLI Score Acc 0.758 STS(cos) 0.651 ``` --- layout: model title: BERT Sentence Embeddings trained on MEDLINE/PubMed and fine-tuned on SQuAD 2.0 author: John Snow Labs name: sent_bert_pubmed_squad2 date: 2021-08-31 tags: [en, open_source, sentence_embeddings, medline_pubmed_dataset, squad_2_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/pubmed/1 and fine-tuned on SQuAD 2.0. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. This model is intended to be used for a variety of English NLP tasks in the medical domain. This model is fine-tuned on the SQuAD 2.0 as a span-labeling task to label the answer to a question in a given context and is recommended for use in question answering tasks. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_pubmed_squad2_en_3.2.0_3.0_1630412086842.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_pubmed_squad2_en_3.2.0_3.0_1630412086842.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_pubmed_squad2", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_pubmed_squad2", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.pubmed_squad2').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_pubmed_squad2| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [MEDLINE/PubMed dataset](https://www.nlm.nih.gov/databases/download/pubmed_medline.html) [2]: [Stanford Queston Answering (SQuAD 2.0) dataset](https://rajpurkar.github.io/SQuAD-explorer/) This Model has been imported from: https://tfhub.dev/google/experts/bert/pubmed/squad2/2 --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Russian (WikiNER 840B 300) author: John Snow Labs name: wikiner_840B_300 date: 2020-03-16 task: Named Entity Recognition language: ru edition: Spark NLP 2.4.4 spark_version: 2.4 tags: [ner, ru, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 840B 300 is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_RU){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_RU.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_ru_2.4.4_2.4_1584014001695.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_ru_2.4.4_2.4_1584014001695.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", lang="xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = NerDLModel.pretrained("wikiner_840B_300", "ru") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", lang="xx") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("wikiner_840B_300", "ru") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла."""] ner_df = nlu.load('ru.ner.wikiner.glove.840B_300').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Уильям Генри Гейтс III|PER | |Microsoft |ORG | |Microsoft Гейтс |ORG | |CEO |ORG | |Гейтс |PER | |Сиэтле |LOC | |Вашингтон |LOC | |Полом Алленом |PER | |Альбукерке |LOC | |Нью-Мексико |LOC | |Microsoft |ORG | |Гейтс |PER | |Гейтс |PER | |Гейтс |PER | |Microsoft |ORG | |Фонде Билла |PER | |Мелинды Гейтс |PER | |Мелинда Гейтс |PER | |Постепенно |PER | |Рэю Оззи |PER | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ru| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://ru.wikipedia.org](https://ru.wikipedia.org) --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Original_BlueBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Original-BlueBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_BlueBERT_384_en_4.0.0_3.0_1657108595543.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_BlueBERT_384_en_4.0.0_3.0_1657108595543.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Original_BlueBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Original-BlueBERT-384 --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_original_PubmedBert date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4-original-PubmedBert` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_original_PubmedBert_en_4.0.0_3.0_1657108214734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_original_PubmedBert_en_4.0.0_3.0_1657108214734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_original_PubmedBert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4-original-PubmedBert --- layout: model title: Word Embeddings for Arabic (arabic_w2v_cc_300d) author: John Snow Labs name: arabic_w2v_cc_300d date: 2020-12-05 task: Embeddings language: ar edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [embeddings, ar, open_source] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/arabic_w2v_cc_300d_ar_2.7.0_2.4_1607168354606.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/arabic_w2v_cc_300d_ar_2.7.0_2.4_1607168354606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['أنا أحب التعلم الآلي']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("أنا أحب التعلم الآلي").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["أنا أحب التعلم الآلي"] arabicvec_df = nlu.load('ar.embed.cbow.300d').predict(text, output_level='token') arabicvec_df ```

{:.h2_title} ## Results The model gives 300 dimensional Word2Vec feature vector outputs per token. ```bash | ar_embed_cbow_300d_embeddings token |----------------------------------------------------|-------- | [-0.11158058792352676, -0.06634224951267242, -... أنا | [-0.2818698585033417, -0.21061033010482788, -0... أحب ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|arabic_w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|ar| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from [https://fasttext.cc/docs/en/crawl-vectors.html](https://fasttext.cc/docs/en/crawl-vectors.html) --- layout: model title: English DistilBertForQuestionAnswering model (from jgammack) SAE author: John Snow Labs name: distilbert_qa_SAE_base_uncased_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `SAE-distilbert-base-uncased-squad` is a English model originally trained by `jgammack`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_SAE_base_uncased_squad_en_4.0.0_3.0_1654722995068.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_SAE_base_uncased_squad_en_4.0.0_3.0_1654722995068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_SAE_base_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_SAE_base_uncased_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased_sae.by_jgammack").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_SAE_base_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/jgammack/SAE-distilbert-base-uncased-squad --- layout: model title: Legal Definitions Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_definitions_bert date: 2023-03-05 tags: [en, legal, classification, clauses, definitions, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Definitions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Definitions`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_definitions_bert_en_1.0.0_3.0_1678049915628.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_definitions_bert_en_1.0.0_3.0_1678049915628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_definitions_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Definitions]| |[Other]| |[Other]| |[Definitions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_definitions_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Definitions 0.97 0.96 0.96 67 Other 0.97 0.98 0.97 95 accuracy - - 0.97 162 macro-avg 0.97 0.97 0.97 162 weighted-avg 0.97 0.97 0.97 162 ``` --- layout: model title: Relation Extraction between anatomical entities and other clinical entities (ReDL) author: John Snow Labs name: redl_oncology_location_biobert_wip date: 2023-01-15 tags: [licensed, clinical, oncology, en, relation_extraction, anatomy, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links extractions from anatomical entities (such as Site_Breast or Site_Lung) to other clinical entities (such as Tumor_Finding or Cancer_Surgery). ## Predicted Entities `is_location_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_location_biobert_wip_en_4.2.4_3.0_1673770597615.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_location_biobert_wip_en_4.2.4_3.0_1673770597615.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash +--------------+-------------+-------------+-----------+------+-----------+-------------+-----------+------+----------+ | relation| entity1|entity1_begin|entity1_end|chunk1| entity2|entity2_begin|entity2_end|chunk2|confidence| +--------------+-------------+-------------+-----------+------+-----------+-------------+-----------+------+----------+ |is_location_of|Tumor_Finding| 35| 38| lump|Site_Breast| 53| 58|breast| 0.9628376| +--------------+-------------+-------------+-----------+------+-----------+-------------+-----------+------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_location_biobert_wip| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.90 0.94 0.92 is_location_of 0.94 0.90 0.92 macro-avg 0.92 0.92 0.92 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_rbt6 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rbt6` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt6_zh_4.2.4_3.0_1670327118775.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt6_zh_4.2.4_3.0_1670327118775.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt6","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt6","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt6| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|224.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt6 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Legal Cause Clause Binary Classifier author: John Snow Labs name: legclf_cause_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `cause` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `cause` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cause_clause_en_1.0.0_3.2_1660122210522.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cause_clause_en_1.0.0_3.2_1660122210522.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cause_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[cause]| |[other]| |[other]| |[cause]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cause_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support cause 0.96 1.00 0.98 27 other 1.00 0.99 1.00 108 accuracy - - 0.99 135 macro-avg 0.98 1.00 0.99 135 weighted-avg 0.99 0.99 0.99 135 ``` --- layout: model title: Chinese BertForQuestionAnswering model (from jackh1995) author: John Snow Labs name: bert_qa_bert_chinese_finetuned date: 2022-06-02 tags: [zh, open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-chinese-finetuned` is a Chinese model orginally trained by `jackh1995`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_chinese_finetuned_zh_4.0.0_3.0_1654181635362.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_chinese_finetuned_zh_4.0.0_3.0_1654181635362.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_chinese_finetuned","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_chinese_finetuned","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.by_jackh1995").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_chinese_finetuned| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|381.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jackh1995/bert-chinese-finetuned --- layout: model title: Stopwords Remover for Ancient Greek language (907 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, grc, open_source] task: Stop Words Removal language: grc edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_grc_3.4.1_3.0_1646673167002.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_grc_3.4.1_3.0_1646673167002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------+ |result | +---------------------------------------------------+ |[Όντας, δε, θνητούς, θνητά, και, φρονείν, χρεών, .]| +---------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|grc| |Size:|4.5 KB| --- layout: model title: English BertForQuestionAnswering model (from rsvp-ai) author: John Snow Labs name: bert_qa_bertserini_bert_base_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertserini-bert-base-squad` is a English model orginally trained by `rsvp-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bertserini_bert_base_squad_en_4.0.0_3.0_1654185449571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bertserini_bert_base_squad_en_4.0.0_3.0_1654185449571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bertserini_bert_base_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bertserini_bert_base_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base.by_rsvp-ai").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bertserini_bert_base_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rsvp-ai/bertserini-bert-base-squad --- layout: model title: Google's Tapas Table Understanding (Mini, WTQ) author: John Snow Labs name: table_qa_tapas_mini_finetuned_wtq date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Mini Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_mini_finetuned_wtq_en_4.2.0_3.0_1664530449660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_mini_finetuned_wtq_en_4.2.0_3.0_1664530449660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_mini_finetuned_wtq","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_efi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_timit_demo_colab11_by_sameearif88 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab11_by_sameearif88 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab11_by_sameearif88` is a English model originally trained by sameearif88. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_en_4.2.0_3.0_1664021285479.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_en_4.2.0_3.0_1664021285479.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab11_by_sameearif88", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab11_by_sameearif88", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab11_by_sameearif88| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English AlbertForQuestionAnswering model (from madlag) author: John Snow Labs name: albert_qa_base_v2_squad date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `albert-base-v2-squad` is a English model originally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_base_v2_squad_en_4.0.0_3.0_1656063705520.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_base_v2_squad_en_4.0.0_3.0_1656063705520.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_base_v2_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_base_v2_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.albert.base_v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_base_v2_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/madlag/albert-base-v2-squad - https://github.com/google-research/albert --- layout: model title: Extract relations between problem, test, and findings in reports author: John Snow Labs name: re_test_problem_finding date: 2021-04-19 tags: [en, relation_extraction, licensed, clinical] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.1 spark_version: 2.4 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Find relations between diagnosis, tests and imaging findings in radiology reports. `1` : The two entities are related. `0` : The two entities are not related ## Predicted Entities `0`, `1` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_RADIOLOGY/){:.button.button-orange} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_test_problem_finding_en_2.7.1_2.4_1618830922197.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_test_problem_finding_en_2.7.1_2.4_1618830922197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `re_test_problem_finding` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:-----------------------:|:--------------:|:---------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | re_test_problem_finding | 0,1 | ner_jsl | [“test-cerebrovascular_disease”,
“cerebrovascular_disease-test”,
“test-communicable_disease”,
“communicable_disease-test”,
“test-diabetes”, “diabetes-test”,
“test-disease_syndrome_disorder”,
“disease_syndrome_disorder-test”,
“test-heart_disease”,
“heart_disease-test”,
“test-hyperlipidemia”,
“hyperlipidemia-test”,
“test-hypertension”,
“hypertension-test”,
“test-injury_or_poisoning”,
“injury_or_poisoning-test”,
“test-kidney_disease”,
“kidney_disease-test”,
“test-obesity”,
“obesity-test”,
“test-oncological”,
“oncological-test”,
“test-psychological_condition”,
“psychological_condition-test”,
“test-symptom”, “symptom-test”,
“ekg_findings-disease_syndrome_disorder”,
“disease_syndrome_disorder-ekg_findings”,
“ekg_findings-heart_disease”,
“heart_disease-ekg_findings”,
“ekg_findings-symptom”,
“symptom-ekg_findings”,
“imagingfindings-cerebrovascular_disease”,
“cerebrovascular_disease-imagingfindings”,
“imagingfindings-communicable_disease”,
“communicable_disease-imagingfindings”,
“imagingfindings-disease_syndrome_disorder”,
“disease_syndrome_disorder-imagingfindings”,
“imagingfindings-heart_disease”,
“heart_disease-imagingfindings”,
“imagingfindings-hyperlipidemia”,
“hyperlipidemia-imagingfindings”,
“imagingfindings-hypertension”,
“hypertension-imagingfindings”,
“imagingfindings-injury_or_poisoning”,
“injury_or_poisoning-imagingfindings”,
“imagingfindings-kidney_disease”,
“kidney_disease-imagingfindings”,
“imagingfindings-oncological”,
“oncological-imagingfindings”,
“imagingfindings-psychological_condition”,
“psychological_condition-imagingfindings”,
“imagingfindings-symptom”,
“symptom-imagingfindings”,
“vs_finding-cerebrovascular_disease”,
“cerebrovascular_disease-vs_finding”,
“vs_finding-communicable_disease”,
“communicable_disease-vs_finding”,
“vs_finding-diabetes”,
“diabetes-vs_finding”,
“vs_finding-disease_syndrome_disorder”,
“disease_syndrome_disorder-vs_finding”,
“vs_finding-heart_disease”,
“heart_disease-vs_finding”,
“vs_finding-hyperlipidemia”,
“hyperlipidemia-vs_finding”,
“vs_finding-hypertension”,
“hypertension-vs_finding”,
“vs_finding-injury_or_poisoning”,
“injury_or_poisoning-vs_finding”,
“vs_finding-kidney_disease”,
“kidney_disease-vs_finding”,
“vs_finding-obesity”,
“obesity-vs_finding”,
“vs_finding-oncological”,
“oncological-vs_finding”,
“vs_finding-overweight”,
“overweight-vs_finding”,
“vs_finding-psychological_condition”,
“psychological_condition-vs_finding”,
“vs_finding-symptom”, “symptom-vs_finding”] |

## Results ```bash | index | relations | entity1 | chunk1 | entity2 | chunk2 | |-------|--------------|--------------|---------------------|--------------|---------| | 0 | 1 | PROCEDURE | biopsy | SYMPTOM | lesion | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_test_problem_finding| |Type:|re| |Compatibility:|Healthcare NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| ## Data Source Trained on internal datasets. --- layout: model title: English BertForQuestionAnswering Cased model (from maroo93) author: John Snow Labs name: bert_qa_kd_squad1.1 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `kd_squad1.1` is a English model originally trained by `maroo93`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_kd_squad1.1_en_4.0.0_3.0_1657189570964.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_kd_squad1.1_en_4.0.0_3.0_1657189570964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_kd_squad1.1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_kd_squad1.1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_kd_squad1.1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|249.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/maroo93/kd_squad1.1 --- layout: model title: Chinese Part of Speech Tagger (from raynardj) author: John Snow Labs name: bert_pos_classical_chinese_punctuation_guwen_biaodian date: 2022-05-09 tags: [bert, pos, part_of_speech, zh, open_source] task: Part of Speech Tagging language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `classical-chinese-punctuation-guwen-biaodian` is a Chinese model orginally trained by `raynardj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_classical_chinese_punctuation_guwen_biaodian_zh_3.4.2_3.0_1652088290238.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_classical_chinese_punctuation_guwen_biaodian_zh_3.4.2_3.0_1652088290238.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_classical_chinese_punctuation_guwen_biaodian| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|zh| |Size:|381.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/raynardj/classical-chinese-punctuation-guwen-biaodian - https://github.com/raynardj/yuan - https://github.com/raynardj/yuan - https://github.com/raynardj/yuan --- layout: model title: English RobertaForQuestionAnswering (from sunitha) author: John Snow Labs name: roberta_qa_Roberta_Custom_Squad_DS date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Roberta_Custom_Squad_DS` is a English model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_Roberta_Custom_Squad_DS_en_4.0.0_3.0_1655727273046.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_Roberta_Custom_Squad_DS_en_4.0.0_3.0_1655727273046.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_Roberta_Custom_Squad_DS","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_Roberta_Custom_Squad_DS","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.by_sunitha").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_Roberta_Custom_Squad_DS| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/Roberta_Custom_Squad_DS --- layout: model title: Ukrainian BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_uk_cased date: 2022-12-02 tags: [uk, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: uk edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uk-cased` is a Ukrainian model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uk_cased_uk_4.2.4_3.0_1670019147754.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uk_cased_uk_4.2.4_3.0_1670019147754.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_uk_cased","uk") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_uk_cased","uk") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_uk_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|uk| |Size:|357.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-uk-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate English to San Salvador Kongo Pipeline author: John Snow Labs name: translate_en_kwy date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, kwy, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `kwy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_kwy_xx_2.7.0_2.4_1609688437952.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_kwy_xx_2.7.0_2.4_1609688437952.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_kwy", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_kwy", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.kwy').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_kwy| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Greek BertForQuestionAnswering Cased model (from Danastos) author: John Snow Labs name: bert_qa_qacombination_el_4 date: 2022-07-07 tags: [el, open_source, bert, question_answering] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qacombination_bert_el_4` is a Greek model originally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_qacombination_el_4_el_4.0.0_3.0_1657190786453.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_qacombination_el_4_el_4.0.0_3.0_1657190786453.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_qacombination_el_4","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_qacombination_el_4","el") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_qacombination_el_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|el| |Size:|421.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/qacombination_bert_el_4 --- layout: model title: Company Name Normalization using Nasdaq Stock Screener author: John Snow Labs name: finel_nasdaq_company_name_stock_screener date: 2023-01-20 tags: [en, finance, licensed, nasdaq, company] task: Entity Resolution language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Financial Entity Resolver model, trained to obtain normalized versions of Company Names, registered in NASDAQ Stock Screener. You can use this model after extracting a company name using any NER, and you will obtain the official name of the company as per NASDAQ Stock Screener. After this, you can use `finmapper_nasdaq_company_name_stock_screener` to augment and obtain more information about a company using NASDAQ Stock Screener, including Ticker, Sector, Country, etc. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finel_nasdaq_company_name_stock_screener_en_1.0.0_3.0_1674233034536.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finel_nasdaq_company_name_stock_screener_en_1.0.0_3.0_1674233034536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained("finner_orgs_prods_alias", "en", "finance/models")\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") chunkToDoc = nlp.Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") chunk_embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk_doc") \ .setOutputCol("sentence_embeddings") use_er_model = finance.SentenceEntityResolverModel.pretrained("finel_nasdaq_company_name_stock_screener", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("normalized")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner_model, ner_converter, chunkToDoc, chunk_embeddings, use_er_model ]) text = """NIKE is an American multinational corporation that is engaged in the design, development, manufacturing, and worldwide marketing and sales of footwear, apparel, equipment, accessories, and services.""" test_data = spark.createDataFrame([[text]]).toDF("text") model = nlpPipeline.fit(test_data) lp = nlp.LightPipeline(model) result = lp.annotate(text) result["normalized"] ```

## Results ```bash ['Nike Inc. Common Stock'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finel_nasdaq_company_name_stock_screener| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[normalized]| |Language:|en| |Size:|54.7 MB| |Case sensitive:|false| ## References https://www.nasdaq.com/market-activity/stocks/screener --- layout: model title: Self Reported Stress Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_self_reported_stress_tweet date: 2022-07-29 tags: [en, licenced, clinical, public_health, sequence_classification, classifier, stress, licensed] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true recommended: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT based](https://github.com/dmis-lab/biobert) classifier that can identify stress in social media (Twitter) posts in the self-disclosure category. The model finds whether a person claims he/she is stressed or not. ## Predicted Entities `not-stressed`, `stressed` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_STRESS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_self_reported_stress_tweet_en_4.0.0_3.0_1659087442993.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_self_reported_stress_tweet_en_4.0.0_3.0_1659087442993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_self_reported_stress_tweet", "en", "clinical/models")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) data = spark.createDataFrame([["Do you feel stressed?"], ["I'm so stressed!"], ["Depression and anxiety will probably end up killing me – I feel so stressed all the time and just feel awful."], ["Do you enjoy living constantly in this self-inflicted stress?"]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("text", "class.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_self_reported_stress_tweet", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val data = Seq(Array("Do you feel stressed!", "I'm so stressed!", "Depression and anxiety will probably end up killing me – I feel so stressed all the time and just feel awful.", "Do you enjoy living constantly in this self-inflicted stress?")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.self_reported_stress").predict("""Depression and anxiety will probably end up killing me – I feel so stressed all the time and just feel awful.""") ```

## Results ```bash +-------------------------------------------------------------------------------------------------------------+--------------+ |text |result | +-------------------------------------------------------------------------------------------------------------+--------------+ |Do you feel stressed? |[not-stressed]| |I'm so stressed! |[stressed] | |Depression and anxiety will probably end up killing me – I feel so stressed all the time and just feel awful.|[stressed] | |Do you enjoy living constantly in this self-inflicted stress? |[not-stressed]| +-------------------------------------------------------------------------------------------------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_self_reported_stress_tweet| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## Benchmarking ```bash label precision recall f1-score support not-stressed 0.8564 0.8020 0.8283 409 stressed 0.7197 0.7909 0.7536 263 accuracy - - 0.7976 672 macro-avg 0.7881 0.7964 0.7910 672 weighted-avg 0.8029 0.7976 0.7991 672 ``` --- layout: model title: English asr_wav2vec2_large_960h_lv60_self_4_gram TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: asr_wav2vec2_large_960h_lv60_self_4_gram date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h_lv60_self_4_gram` is a English model originally trained by patrickvonplaten. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_960h_lv60_self_4_gram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_lv60_self_4_gram_en_4.2.0_3.0_1664021695681.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_lv60_self_4_gram_en_4.2.0_3.0_1664021695681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_960h_lv60_self_4_gram", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_960h_lv60_self_4_gram", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_960h_lv60_self_4_gram| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|757.4 MB| --- layout: model title: Turkish XlmRoBertaForQuestionAnswering (from Aybars) author: John Snow Labs name: xlm_roberta_qa_XLM_Turkish date: 2022-06-23 tags: [tr, open_source, question_answering, xlmroberta] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `XLM_Turkish` is a Turkish model originally trained by `Aybars`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_XLM_Turkish_tr_4.0.0_3.0_1655983903393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_XLM_Turkish_tr_4.0.0_3.0_1655983903393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_XLM_Turkish","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_XLM_Turkish","tr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_XLM_Turkish| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|tr| |Size:|792.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Aybars/XLM_Turkish --- layout: model title: English asr_wav2vec2_coral_300ep TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: pipeline_asr_wav2vec2_coral_300ep date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_coral_300ep` is a English model originally trained by joaoalvarenga. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_coral_300ep_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_coral_300ep_en_4.2.0_3.0_1664023766656.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_coral_300ep_en_4.2.0_3.0_1664023766656.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_coral_300ep', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_coral_300ep", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_coral_300ep| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline for Detect Subentity PHI for Deidentification (Arabic) author: John Snow Labs name: ner_deid_subentity_pipeline date: 2023-05-31 tags: [licensed, clinical, deidentification, ar, pipeline] task: Pipeline Healthcare language: ar edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_subentity](https://nlp.johnsnowlabs.com/2023/05/29/ner_deid_subentity_ar.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pipeline_ar_4.4.1_3.0_1685563688023.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pipeline_ar_4.4.1_3.0_1685563688023.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_subentity_pipeline", "ar", "clinical/models") text= ''' ملاحظات سريرية - مريض الربو. التاريخ: 16 أبريل 2000. اسم المريضة: ليلى حسن. العنوان: شارع المعرفة، مبنى رقم 789، حي الأمانة، جدة. الرمز البريدي: 54321. البلد: المملكة العربية السعودية. اسم المستشفى: مستشفى النور. اسم الطبيب: د. أميرة أحمد. تفاصيل الحالة: المريضة ليلى حسن، البالغة من العمر 35 عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح. ''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_subentity_pipeline", "ar", "clinical/models") val text = "ملاحظات سريرية - مريض الربو. التاريخ: 16 أبريل 2000. اسم المريضة: ليلى حسن. العنوان: شارع المعرفة، مبنى رقم 789، حي الأمانة، جدة. الرمز البريدي: 54321. البلد: المملكة العربية السعودية. اسم المستشفى: مستشفى النور. اسم الطبيب: د. أميرة أحمد. تفاصيل الحالة: المريضة ليلى حسن، البالغة من العمر 35 عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح." val result = pipeline.fullAnnotate(text) ```

## Results ```bash +---------------+--------+ |chunks |entities| +---------------+--------+ |16 أبريل 2000 |DATE | |ليلى حسن |PATIENT | |789، |ZIP | |جدة |CITY | |54321 |ZIP | |المملكة العربية|CITY | |السعودية |COUNTRY | |النور |HOSPITAL| |أميرة أحمد |DOCTOR | |ليلى |PATIENT | |35 |AGE | +---------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Language:|ar| |Size:|1.2 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English XLMRobertaForTokenClassification Base Cased model (from tner) author: John Snow Labs name: xlmroberta_ner_base_bc5cdr date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-bc5cdr` is a English model originally trained by `tner`. ## Predicted Entities `chemical`, `disease` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_bc5cdr_en_4.1.0_3.0_1660425851127.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_bc5cdr_en_4.1.0_3.0_1660425851127.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_bc5cdr","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_bc5cdr","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_bc5cdr| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|780.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/tner/xlm-roberta-base-bc5cdr - https://github.com/asahi417/tner --- layout: model title: Legal Corporate existence Clause Binary Classifier author: John Snow Labs name: legclf_corporate_existence_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `corporate-existence` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `corporate-existence` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_corporate_existence_clause_en_1.0.0_3.2_1660123366726.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_corporate_existence_clause_en_1.0.0_3.2_1660123366726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_corporate_existence_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[corporate-existence]| |[other]| |[other]| |[corporate-existence]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_corporate_existence_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support corporate-existence 0.91 0.93 0.92 43 other 0.96 0.95 0.95 76 accuracy - - 0.94 119 macro-avg 0.93 0.94 0.94 119 weighted-avg 0.94 0.94 0.94 119 ``` --- layout: model title: Bangla BertForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: bert_embeddings_indic_transformers date: 2022-12-06 tags: [bn, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: bn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-bn-bert` is a Bangla model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_bn_4.2.4_3.0_1670326563595.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_bn_4.2.4_3.0_1670326563595.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","bn") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","bn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_indic_transformers| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|505.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-bn-bert - https://oscar-corpus.com/ --- layout: model title: Extract Cancer Therapies and Granular Posology Information author: John Snow Labs name: ner_oncology_posology date: 2022-11-24 tags: [licensed, clinical, en, oncology, ner, treatment, posology] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts cancer therapies (Cancer_Surgery, Radiotherapy and Cancer_Therapy) and posology information at a granular level. Definitions of Predicted Entities: - `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. - `Cancer_Therapy`: Any cancer treatment mentioned in text, excluding surgeries and radiotherapy. - `Cycle_Count`: The total number of cycles being administered of an oncological therapy (e.g. "5 cycles"). - `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. "day 5"). - `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. "third cycle"). - `Dosage`: The quantity prescribed by the physician for an active ingredient. - `Duration`: Words indicating the duration of a treatment (e.g. "for 2 weeks"). - `Frequency`: Words indicating the frequency of treatment administration (e.g. "daily" or "bid"). - `Radiotherapy`: Terms that indicate the use of Radiotherapy. - `Radiation_Dose`: Dose used in radiotherapy. - `Route`: Words indicating the type of administration route (such as "PO" or "transdermal"). ## Predicted Entities `Cancer_Surgery`, `Cancer_Therapy`, `Cycle_Count`, `Cycle_Day`, `Cycle_Number`, `Dosage`, `Duration`, `Frequency`, `Radiotherapy`, `Radiation_Dose`, `Route` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_posology_en_4.2.2_3.0_1669306988706.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_posology_en_4.2.2_3.0_1669306988706.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:---------------| | adriamycin | Cancer_Therapy | | 60 mg/m2 | Dosage | | cyclophosphamide | Cancer_Therapy | | 600 mg/m2 | Dosage | | six courses | Cycle_Count | | second cycle | Cycle_Number | | chemotherapy | Cancer_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_posology| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Cycle_Number 52 4 45 97 0.93 0.54 0.68 Cycle_Count 200 63 30 230 0.76 0.87 0.81 Radiotherapy 255 16 55 310 0.94 0.82 0.88 Cancer_Surgery 592 66 227 819 0.90 0.72 0.80 Cycle_Day 175 22 73 248 0.89 0.71 0.79 Frequency 337 44 90 427 0.88 0.79 0.83 Route 53 1 60 113 0.98 0.47 0.63 Cancer_Therapy 1448 81 250 1698 0.95 0.85 0.90 Duration 525 154 236 761 0.77 0.69 0.73 Dosage 858 79 202 1060 0.92 0.81 0.86 Radiation_Dose 86 4 40 126 0.96 0.68 0.80 macro_avg 4581 534 1308 5889 0.90 0.72 0.79 micro_avg 4581 534 1308 5889 0.90 0.78 0.83 ``` --- layout: model title: Indonesian T5ForConditionalGeneration Base Cased model (from Wikidepia) author: John Snow Labs name: t5_indot5_base_paraphrase date: 2023-01-30 tags: [id, open_source, t5] task: Text Generation language: id edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `IndoT5-base-paraphrase` is a Indonesian model originally trained by `Wikidepia`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_indot5_base_paraphrase_id_4.3.0_3.0_1675097776595.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_indot5_base_paraphrase_id_4.3.0_3.0_1675097776595.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_indot5_base_paraphrase","id") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_indot5_base_paraphrase","id") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_indot5_base_paraphrase| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|id| |Size:|1.0 GB| ## References - https://huggingface.co/Wikidepia/IndoT5-base-paraphrase --- layout: model title: Оcr small for printed text author: John Snow Labs name: ocr_small_printed date: 2022-02-16 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 3.3.3 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Ocr small model for recognise printed text based on TrOcr architecture. The TrOCR model was proposed in TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. TrOCR consists of an image Transformer encoder and an autoregressive text Transformer decoder to perform optical character recognition (OCR). The abstract from the paper is the following: Text recognition is a long-standing research problem for document digitalization. Existing approaches for text recognition are usually built based on CNN for image understanding and RNN for char-level text generation. In addition, another language model is usually needed to improve the overall accuracy as a post-processing step. In this paper, we propose an end-to-end text recognition approach with pre-trained image Transformer and text Transformer models, namely TrOCR, which leverages the Transformer architecture for both image understanding and wordpiece-level text generation. The TrOCR model is simple but effective, and can be pre-trained with large-scale synthetic data and fine-tuned with human-labeled datasets. Experiments show that the TrOCR model outperforms the current state-of-the-art models on both printed and handwritten text recognition tasks. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_printed_en_3.3.3_2.4_1645007455031.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_printed_en_3.3.3_2.4_1645007455031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ocr = ImageToTextv2().pretrained("ocr_small_printed", "en", "clinical/ocr") ocr.setInputCols(["image"]) ocr.setOutputCol("text") result = ocr.transform(image_text_lines_df).collect() print(result[0].text)

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ocr = ImageToTextv2().pretrained("ocr_base_printed", "en", "clinical/ocr") ocr.setInputCols(["image"]) ocr.setOutputCol("text") result = ocr.transform(image_text_lines_df).collect() print(result[0].text)

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ocr = ImageToTextv2().pretrained("ocr_small_printed", "en", "clinical/ocr") ocr.setInputCols(["image"]) ocr.setOutputCol("text") result = ocr.transform(image_text_lines_df).collect() print(result[0].text)

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ocr_small_printed| |Type:|ocr| |Compatibility:|Visual NLP 3.3.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|146.7 MB| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab9 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab9` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab9_en_4.2.0_3.0_1664019939623.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab9_en_4.2.0_3.0_1664019939623.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab9", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab9", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Spanish RobertaForQuestionAnswering (from jamarju) author: John Snow Labs name: roberta_qa_roberta_base_bne_squad_2.0_es_jamarju date: 2022-06-21 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-squad-2.0-es` is a Spanish model originally trained by `jamarju`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_bne_squad_2.0_es_jamarju_es_4.0.0_3.0_1655789380928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_bne_squad_2.0_es_jamarju_es_4.0.0_3.0_1655789380928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_bne_squad_2.0_es_jamarju","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_bne_squad_2.0_es_jamarju","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.roberta.base.by_jamarju").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_bne_squad_2.0_es_jamarju| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|456.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jamarju/roberta-base-bne-squad-2.0-es - https://github.com/PlanTL-SANIDAD/lm-spanish - https://github.com/ccasimiro88/TranslateAlignRetrieve --- layout: model title: English DistilBertForQuestionAnswering model (from Hoang) author: John Snow Labs name: distilbert_qa_Hoang_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Hoang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Hoang_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724211596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Hoang_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724211596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Hoang_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Hoang_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Hoang").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Hoang_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Hoang/distilbert-base-uncased-finetuned-squad --- layout: model title: ICD10GM ChunkResolver author: John Snow Labs name: chunkresolve_ICD10GM class: ChunkEntityResolverModel language: de repository: clinical/models date: 2020-09-06 task: Entity Resolution edition: Healthcare NLP 2.5.5 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,de] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/14.German_Healthcare_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_ICD10GM_de_2.5.5_2.4_1599431635423.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_ICD10GM_de_2.5.5_2.4_1599431635423.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... icd10_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_ICD10GM",'de','clinical/models') \ .setInputCols(["token", "chunk_embeddings"]) \ .setOutputCol("icd10_de_code")\ .setDistanceFunction("EUCLIDEAN") \ .setNeighbours(5) pipeline_icd10 = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, de_embeddings, de_ner, ner_converter, chunk_embeddings, icd10_resolution]) empty_data = spark.createDataFrame([['''Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist Hernia femoralis, Akne, einseitig, ein hochmalignes bronchogenes Karzinom, das überwiegend im Zentrum der Lunge, in einem Hauptbronchus entsteht. Die mittlere Prävalenz wird auf 1/20.000 geschätzt. Vom SCLC sind hauptsächlich Peronen mittleren Alters (27-66 Jahre) mit Raucheranamnese betroffen. Etwa 70% der Patienten mit SCLC haben bei Stellung der Diagnose schon extra-thorakale Symptome. Zu den Symptomen gehören Thoraxschmerz, Dyspnoe, Husten und pfeifende Atmung. Die Beteiligung benachbarter Bereiche verursacht Heiserkeit, Dysphagie und Oberes Vena-cava-Syndrom (Obstruktion des Blutflusses durch die Vena cava superior). Zusätzliche Symptome als Folge einer Fernmetastasierung sind ebenfalls möglich. Rauchen und Strahlenexposition sind synergistisch wirkende Risikofaktoren. Die industrielle Exposition mit Bis (Chlormethyläther) ist ein weiterer Risikofaktor. Röntgenaufnahmen des Thorax sind nicht ausreichend empfindlich, um einen SCLC frühzeitig zu erkennen. Röntgenologischen Auffälligkeiten muß weiter nachgegangen werden, meist mit Computertomographie. Die Diagnose wird bioptisch gesichert. Patienten mit SCLC erhalten meist Bestrahlung und/oder Chemotherapie. In Hinblick auf eine Verbesserung der Überlebenschancen der Patienten ist sowohl bei ausgedehnten und bei begrenzten SCLC eine kombinierte Chemotherapie wirksamer als die Behandlung mit Einzelsubstanzen. Es kann auch eine prophylaktische Bestrahlung des Schädels erwogen werden, da innerhalb von 2-3 Jahren nach Behandlungsbeginn ein hohes Risiko für zentralnervöse Metastasen besteht. Das Kleinzellige Bronchialkarzinom ist der aggressivste Lungentumor: Die 5-Jahres-Überlebensrate beträgt 1-5%, der Median des gesamten Überlebens liegt bei etwa 6 bis 10 Monaten.''']]).toDF("text") model = pipeline_icd10.fit(empty_data) results = model.transform(data) ``` ```scala ... val icd10_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_ICD10GM",'de','clinical/models') .setInputCols("token", "chunk_embeddings") .setOutputCol("icd10_de_code") .setDistanceFunction("EUCLIDEAN") .setNeighbours(5) val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, de_embeddings, de_ner, ner_converter, chunk_embeddings, icd10_resolution)) val data = Seq("Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist Hernia femoralis, Akne, einseitig, ein hochmalignes bronchogenes Karzinom, das überwiegend im Zentrum der Lunge, in einem Hauptbronchus entsteht. Die mittlere Prävalenz wird auf 1/20.000 geschätzt. Vom SCLC sind hauptsächlich Peronen mittleren Alters (27-66 Jahre) mit Raucheranamnese betroffen. Etwa 70% der Patienten mit SCLC haben bei Stellung der Diagnose schon extra-thorakale Symptome. Zu den Symptomen gehören Thoraxschmerz, Dyspnoe, Husten und pfeifende Atmung. Die Beteiligung benachbarter Bereiche verursacht Heiserkeit, Dysphagie und Oberes Vena-cava-Syndrom (Obstruktion des Blutflusses durch die Vena cava superior). Zusätzliche Symptome als Folge einer Fernmetastasierung sind ebenfalls möglich. Rauchen und Strahlenexposition sind synergistisch wirkende Risikofaktoren. Die industrielle Exposition mit Bis (Chlormethyläther) ist ein weiterer Risikofaktor. Röntgenaufnahmen des Thorax sind nicht ausreichend empfindlich, um einen SCLC frühzeitig zu erkennen. Röntgenologischen Auffälligkeiten muß weiter nachgegangen werden, meist mit Computertomographie. Die Diagnose wird bioptisch gesichert. Patienten mit SCLC erhalten meist Bestrahlung und/oder Chemotherapie. In Hinblick auf eine Verbesserung der Überlebenschancen der Patienten ist sowohl bei ausgedehnten und bei begrenzten SCLC eine kombinierte Chemotherapie wirksamer als die Behandlung mit Einzelsubstanzen. Es kann auch eine prophylaktische Bestrahlung des Schädels erwogen werden, da innerhalb von 2-3 Jahren nach Behandlungsbeginn ein hohes Risiko für zentralnervöse Metastasen besteht. Das Kleinzellige Bronchialkarzinom ist der aggressivste Lungentumor: Die 5-Jahres-Überlebensrate beträgt 1-5%, der Median des gesamten Überlebens liegt bei etwa 6 bis 10 Monaten.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash | Problem | ICD10-Code | |--------------------|------------| | Kleinzellige | M01.00 | | Bronchialkarzinom | I50.0 | | Kleinzelliger | I37.0 | | Lungenkrebs | B90.9 | | SCLC | C83.0 | | ... | ... | | Kleinzellige | M01.00 | | Bronchialkarzinom | I50.0 | | Lungentumor | C90.31 | | 1-5% | I37.0 | | 6 bis 10 Monaten | Q91.6 | ``` {:.model-param} ## Model Information {:.table-model} |----------------|--------------------------| | Name: | chunkresolve_ICD10GM | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.5.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | de | | Case sensitive: | True | | Dependencies: | w2v_cc_300d | {:.h2_title} ## Data Source FILLUP --- layout: model title: Onto Recognize Entities Lg author: John Snow Labs name: onto_recognize_entities_lg date: 2022-06-28 tags: [en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The onto_recognize_entities_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entites. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_lg_en_4.0.0_3.0_1656389642706.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_lg_en_4.0.0_3.0_1656389642706.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("onto_recognize_entities_lg", "en") result = pipeline.annotate("""I love johnsnowlabs! """) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.onto.lg").predict("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_lg| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|2.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: Translate Hausa to English Pipeline author: John Snow Labs name: translate_ha_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ha, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ha` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ha_en_xx_2.7.0_2.4_1609686549663.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ha_en_xx_2.7.0_2.4_1609686549663.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ha_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ha_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ha.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ha_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Cancer Genetics (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_bionlp_pipeline date: 2023-03-20 tags: [bertfortokenclassification, ner, bionlp, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_bionlp](https://nlp.johnsnowlabs.com/2022/01/03/bert_token_classifier_ner_bionlp_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bionlp_pipeline_en_4.3.0_3.2_1679308593451.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bionlp_pipeline_en_4.3.0_3.2_1679308593451.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_bionlp_pipeline", "en", "clinical/models") text = '''Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_bionlp_pipeline", "en", "clinical/models") val text = "Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.biolp.pipeline").predict("""Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:-----------------------|-------------:| | 0 | erbA IRES | 9 | 17 | Organism | 0.999188 | | 1 | erbA/myb virus | 27 | 40 | Organism | 0.999434 | | 2 | erythroid cells | 65 | 79 | Cell | 0.999837 | | 3 | bone | 100 | 103 | Multi-tissue_structure | 0.999846 | | 4 | marrow | 105 | 110 | Multi-tissue_structure | 0.999876 | | 5 | blastoderm cultures | 115 | 133 | Cell | 0.999823 | | 6 | erbA/myb IRES virus | 140 | 158 | Organism | 0.999751 | | 7 | erbA IRES virus | 236 | 250 | Organism | 0.999749 | | 8 | blastoderm | 259 | 268 | Cell | 0.999897 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bionlp_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from English to Niger-Kordofanian Languages author: John Snow Labs name: opus_mt_en_nic date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, nic, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `nic` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_nic_xx_2.7.0_2.4_1609167803723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_nic_xx_2.7.0_2.4_1609167803723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_nic| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Cased model (from google) author: John Snow Labs name: t5_efficient_xl_nl4 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-xl-nl4` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_xl_nl4_en_4.3.0_3.0_1675124613893.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_xl_nl4_en_4.3.0_3.0_1675124613893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_xl_nl4","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_xl_nl4","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_xl_nl4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|1.0 GB| ## References - https://huggingface.co/google/t5-efficient-xl-nl4 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Quiet Enjoyment Clause Binary Classifier author: John Snow Labs name: legclf_quiet_enjoyment_clause date: 2023-01-29 tags: [en, legal, classification, quiet, enjoyment, clauses, quiet_enjoyment, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `quiet-enjoyment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `quiet-enjoyment`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_quiet_enjoyment_clause_en_1.0.0_3.0_1675005306234.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_quiet_enjoyment_clause_en_1.0.0_3.0_1675005306234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_quiet_enjoyment_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[quiet-enjoyment]| |[other]| |[other]| |[quiet-enjoyment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_quiet_enjoyment_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 1.00 0.97 39 quiet-enjoyment 1.00 0.94 0.97 33 accuracy - - 0.97 72 macro-avg 0.98 0.97 0.97 72 weighted-avg 0.97 0.97 0.97 72 ``` --- layout: model title: Sentence Entity Resolver for ATC (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_atc date: 2022-03-01 tags: [atc, licensed, en, clinical, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps drugs entities to ATC (Anatomic Therapeutic Chemical) codes using `sbiobert_base_cased_mli ` Sentence Bert Embeddings. ## Predicted Entities `ATC Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_atc_en_3.4.1_3.0_1646126349436.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_atc_en_3.4.1_3.0_1646126349436.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+--------+--------------------------------------------------+--------------------------------------------------+--------------------------------------------------+ | chunk|atc_code| all_k_codes| resolutions| all_k_aux_labels| +-------------------+--------+--------------------------------------------------+--------------------------------------------------+--------------------------------------------------+ | eltrombopag| B02BX05|B02BX05:::A07DA06:::B06AC03:::M01AB08:::L04AA39...|eltrombopag :::eluxadoline :::ecallantide :::et...|ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th...| | amlodipine| C08CA01|C08CA01:::C08CA17:::C08CA13:::C08CA06:::C08CA10...|amlodipine :::levamlodipine :::lercanidipine ::...|ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th...| | metformin| A10BA02|A10BA02:::A10BA01:::A10BB01:::A10BH04:::A10BB07...|metformin :::phenformin :::glyburide / metformi...|ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th...| | hydrogen peroxide| A01AB02|A01AB02:::S02AA06:::D10AE:::D11AX25:::D10AE01::...|hydrogen peroxide :::hydrogen peroxide; otic:::...|ATC 5th:::ATC 5th:::ATC 4th:::ATC 5th:::ATC 5th...| | amoxicillin| J01CA04|J01CA04:::J01CA01:::J01CF02:::J01CF01:::J01CA51...|amoxicillin :::ampicillin :::cloxacillin :::dic...|ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th...| |magnesium hydroxide| A02AA04|A02AA04:::A12CC02:::D10AX30:::B05XA11:::A02AA02...|magnesium hydroxide :::magnesium sulfate :::alu...|ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th:::ATC 5th...| | antidepressant| N06A|N06A:::N05A:::N06AX:::N05AH02:::N06D:::N06CA:::...|ANTIDEPRESSANTS:::ANTIPSYCHOTICS:::Other antide...|ATC 3rd:::ATC 3rd:::ATC 4th:::ATC 5th:::ATC 3rd...| +-------------------+--------+--------------------------------------------------+--------------------------------------------------+--------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_atc| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[atc_code]| |Language:|en| |Size:|71.6 MB| |Case sensitive:|false| ## References Trained on ATC 2022 Codes dataset --- layout: model title: Catalan Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:34:00 +0800 task: Lemmatization language: ca edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, ca] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_ca_2.5.5_2.4_1596054394549.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_ca_2.5.5_2.4_1596054394549.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "ca") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("A part de ser el rei del nord, John Snow és un metge anglès i líder en el desenvolupament de l'anestèsia i la higiene mèdica.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "ca") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("A part de ser el rei del nord, John Snow és un metge anglès i líder en el desenvolupament de l'anestèsia i la higiene mèdica.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""A part de ser el rei del nord, John Snow és un metge anglès i líder en el desenvolupament de l'anestèsia i la higiene mèdica."""] lemma_df = nlu.load('ca.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=0, result='a', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=2, end=5, result='part', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=7, end=8, result='de', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=10, end=12, result='ser', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=14, end=15, result='ell', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|ca| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English BertForQuestionAnswering model (from Nakul24) author: John Snow Labs name: bert_qa_Spanbert_emotion_extraction date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Spanbert-emotion-extraction` is a English model orginally trained by `Nakul24`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Spanbert_emotion_extraction_en_4.0.0_3.0_1654179065087.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Spanbert_emotion_extraction_en_4.0.0_3.0_1654179065087.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Spanbert_emotion_extraction","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Spanbert_emotion_extraction","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.span_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Spanbert_emotion_extraction| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|384.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Nakul24/Spanbert-emotion-extraction --- layout: model title: Pipeline to Detect PHI in medical text (biobert) author: John Snow Labs name: ner_deid_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_biobert_pipeline_en_4.3.0_3.2_1679310594035.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_biobert_pipeline_en_4.3.0_3.2_1679310594035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_biobert_pipeline", "en", "clinical/models") text = '''A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_biobert_pipeline", "en", "clinical/models") val text = "A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.ner_biobert.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D. Name : Hendrickson, Ora MR. # 7194334. PCP : Oliveira, non-smoking. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:------------|-------------:| | 0 | 2093-01-13 | 17 | 26 | DATE | 0.981 | | 1 | David Hale | 29 | 38 | NAME | 0.77585 | | 2 | Hendrickson | 53 | 63 | NAME | 0.9666 | | 3 | Ora | 66 | 68 | LOCATION | 0.8723 | | 4 | Oliveira | 91 | 98 | LOCATION | 0.7785 | | 5 | Cocke County Baptist Hospital | 114 | 142 | LOCATION | 0.792 | | 6 | Keats Street | 150 | 161 | LOCATION | 0.77305 | | 7 | Phone | 164 | 168 | LOCATION | 0.7083 | | 8 | Brothers | 253 | 260 | LOCATION | 0.9447 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: English BertForQuestionAnswering Cased model (from SebastianS) author: John Snow Labs name: bert_qa_sebastians_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `SebastianS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sebastians_finetuned_squad_en_4.0.0_3.0_1657186249406.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sebastians_finetuned_squad_en_4.0.0_3.0_1657186249406.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sebastians_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_sebastians_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sebastians_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SebastianS/bert-finetuned-squad --- layout: model title: Translate English to Pijin Pipeline author: John Snow Labs name: translate_en_pis date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, pis, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `pis` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_pis_xx_2.7.0_2.4_1609698832184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_pis_xx_2.7.0_2.4_1609698832184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_pis", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_pis", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.pis').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_pis| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from English to Tagalog author: John Snow Labs name: opus_mt_en_tl date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tl, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tl_xx_2.7.0_2.4_1609169442130.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tl_xx_2.7.0_2.4_1609169442130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tl| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Western Frisian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, fy, open_source] task: Embeddings language: fy edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_fy_3.4.1_3.0_1647467525855.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_fy_3.4.1_3.0_1647467525855.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_terms_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[terms-agreement]| |[other]| |[other]| |[terms-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_terms_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.95 0.96 65 terms-agreement 0.91 0.94 0.93 34 accuracy - - 0.95 99 macro-avg 0.94 0.95 0.94 99 weighted-avg 0.95 0.95 0.95 99 ``` --- layout: model title: Language Detection & Identification Pipeline - 21 Languages (BiGRU) author: John Snow Labs name: detect_language_bigru_21 date: 2020-12-05 task: [Pipeline Public, Language Detection, Sentence Detection] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, pipeline, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using BiGRU architectures (mentioned in the model's name) in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This pipeline can detect the following languages: ## Predicted Entities `Bulgarian`, `Czech`, `Danish`, `German`, `Greek`, `English`, `Estonian`, `Finnish`, `French`, `Hungarian`, `Italian`, `Lithuanian`, `Latvian`, `Dutch`, `Polish`, `Portuguese`, `Romanian`, `Slovak`, `Slovenian`, `Spanish`, `Swedish`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/LANGUAGE_DETECTOR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/detect_language_bigru_21_xx_2.7.0_2.4_1607186103596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/detect_language_bigru_21_xx_2.7.0_2.4_1607186103596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("detect_language_bigru_21", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("detect_language_bigru_21", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` {:.nlu-block} ```python import nlu text = ["French author who helped pioneer the science-fiction genre."] lang_df = nlu.load("xx.classify.lang.bigru").predict(text) lang_df ```

## Results ```bash {'document': ['French author who helped pioneer the science-fiction genre.'], 'language': ['en']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|detect_language_bigru_21| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - LanguageDetectorDL --- layout: model title: Meena's Tapas Table Understanding (Base) author: John Snow Labs name: table_qa_table_question_answering_tapas date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Base Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_table_question_answering_tapas_en_4.2.0_3.0_1664530457710.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_table_question_answering_tapas_en_4.2.0_3.0_1664530457710.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_table_question_answering_tapas","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

## Results ```bash +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |answer | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |{chunk, 0, 12, Donald Trump, {question -> Who earns less than 200,000,000?, aggregation -> NONE, cell_positions -> [0, 0], cell_scores -> 0.9999999}, []} | |{chunk, 0, 12, Donald Trump, {question -> Who earns 100,000,000?, aggregation -> NONE, cell_positions -> [0, 0], cell_scores -> 0.9999999}, []} | |{chunk, 0, 12, $100,000,000, {question -> How much money has Donald Trump?, aggregation -> NONE, cell_positions -> [1, 0], cell_scores -> 0.9999998}, []} | |{chunk, 0, 6, AVERAGE > 75, 55, {question -> How old are they?, aggregation -> AVERAGE, cell_positions -> [2, 0], [2, 1], cell_scores -> 0.99999976, 0.9999995}, []} | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|table_qa_table_question_answering_tapas| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|413.9 MB| |Case sensitive:|false| ## References https://huggingface.co/models?pipeline_tag=table-question-answering --- layout: model title: Legal Scope Clause Binary Classifier author: John Snow Labs name: legclf_scope_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `scope` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `scope` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_scope_clause_en_1.0.0_3.2_1660123969333.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_scope_clause_en_1.0.0_3.2_1660123969333.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_scope_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[scope]| |[other]| |[other]| |[scope]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_scope_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.90 0.92 88 scope 0.73 0.83 0.77 29 accuracy - - 0.88 117 macro-avg 0.83 0.86 0.85 117 weighted-avg 0.89 0.88 0.88 117 ``` --- layout: model title: Legal Signers Clause Binary Classifier (CUAD dataset) author: John Snow Labs name: legclf_cuad_signers_clause date: 2022-11-17 tags: [signers, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the signers part of a documenttype. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. There are other models in this dataset with similar title, but the difference is the dataset it was trained on. This one was trained with `cuad` dataset. ## Predicted Entities `signers`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_signers_clause_en_1.0.0_3.0_1668693373474.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_signers_clause_en_1.0.0_3.0_1668693373474.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_cuad_signers_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[signers]| |[other]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_signers_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References CUAD dataset ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 73 signers 1.00 1.00 1.00 35 accuracy - - 1.00 108 macro-avg 1.00 1.00 1.00 108 weighted-avg 1.00 1.00 1.00 108 ``` --- layout: model title: English RobertaForQuestionAnswering (from SauravMaheshkar) author: John Snow Labs name: roberta_qa_roberta_base_chaii date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_chaii_en_4.0.0_3.0_1655730347590.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_chaii_en_4.0.0_3.0_1655730347590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/roberta-base-chaii --- layout: model title: Fast Neural Machine Translation Model from Central Bikol to German author: John Snow Labs name: opus_mt_bcl_de date: 2021-06-01 tags: [open_source, seq2seq, translation, bcl, de, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: bcl target languages: de {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_de_xx_3.1.0_2.4_1622550430850.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_de_xx_3.1.0_2.4_1622550430850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bcl_de| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Cebuano (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, ceb, open_source] task: Embeddings language: ceb edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ceb_3.4.1_3.0_1647290267903.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ceb_3.4.1_3.0_1647290267903.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit__beans", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit__beans", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit__beans| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_hier_triplet_shuffled_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223641296.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223641296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_hier_triplet_shuffled_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_hier_triplet_shuffled_epochs_1_shard_1_squad2.0 --- layout: model title: English T5ForConditionalGeneration Cased model (from dbernsohn) author: John Snow Labs name: t5_wikisql_en2sql date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5_wikisql_en2SQL` is a English model originally trained by `dbernsohn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_wikisql_en2sql_en_4.3.0_3.0_1675157192158.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_wikisql_en2sql_en_4.3.0_3.0_1675157192158.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_wikisql_en2sql","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_wikisql_en2sql","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_wikisql_en2sql| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|288.2 MB| ## References - https://huggingface.co/dbernsohn/t5_wikisql_en2SQL - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://github.com/DorBernsohn/CodeLM/tree/main/SQLM - https://www.linkedin.com/in/dor-bernsohn-70b2b1146/ --- layout: model title: Tamil XlmRoBertaForQuestionAnswering (from AswiN037) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_squad_tamil date: 2022-06-23 tags: [ta, open_source, question_answering, xlmroberta] task: Question Answering language: ta edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-squad-tamil` is a Tamil model originally trained by `AswiN037`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_squad_tamil_ta_4.0.0_3.0_1655996786525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_squad_tamil_ta_4.0.0_3.0_1655996786525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_squad_tamil","ta") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_squad_tamil","ta") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ta.answer_question.squad.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_squad_tamil| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|ta| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AswiN037/xlm-roberta-squad-tamil --- layout: model title: English DistilBertForTokenClassification Cased model (from m3hrdadfi) author: John Snow Labs name: distilbert_tok_classifier_typo_detector date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `typo-detector-distilbert-en` is a English model originally trained by `m3hrdadfi`. ## Predicted Entities `TYPO` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_tok_classifier_typo_detector_en_4.3.1_3.0_1678134333311.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_tok_classifier_typo_detector_en_4.3.1_3.0_1678134333311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_tok_classifier_typo_detector","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_tok_classifier_typo_detector","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_tok_classifier_typo_detector| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m3hrdadfi/typo-detector-distilbert-en - https://github.com/neuspell/neuspell - https://github.com/m3hrdadfi/typo-detector/issues --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_roberta_FT_newsqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta_FT_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_FT_newsqa_en_4.0.0_3.0_1655738866363.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_FT_newsqa_en_4.0.0_3.0_1655738866363.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.roberta.qa_roberta_ft_newsqa.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|458.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/roberta_FT_newsqa --- layout: model title: Hebrew BertForQuestionAnswering model (from tdklab) author: John Snow Labs name: bert_qa_hebert_finetuned_hebrew_squad date: 2022-06-02 tags: [he, open_source, question_answering, bert] task: Question Answering language: he edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `hebert-finetuned-hebrew-squad` is a Hebrew model orginally trained by `tdklab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_hebert_finetuned_hebrew_squad_he_4.0.0_3.0_1654187940492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_hebert_finetuned_hebrew_squad_he_4.0.0_3.0_1654187940492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_hebert_finetuned_hebrew_squad","he") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_hebert_finetuned_hebrew_squad","he") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("he.answer_question.squad.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_hebert_finetuned_hebrew_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|he| |Size:|408.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tdklab/hebert-finetuned-hebrew-squad --- layout: model title: English DistilBertForQuestionAnswering model (from FOFer) author: John Snow Labs name: distilbert_qa_FOFer_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `FOFer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_FOFer_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724124682.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_FOFer_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724124682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_FOFer_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_FOFer_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_FOFer").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_FOFer_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/FOFer/distilbert-base-uncased-finetuned-squad --- layout: model title: Word Segmenter for Chinese author: John Snow Labs name: wordseg_ctb9 date: 2021-03-08 tags: [word_segmentation, open_source, chinese, wordseg_ctb9, zh] task: Word Segmentation language: zh edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [WordSegmenterModel-WSM](https://en.wikipedia.org/wiki/Text_segmentation) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know a priori which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/chinese/word_segmentation/words_segmenter_demo.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_ctb9_zh_3.0.0_3.0_1615225768619.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_ctb9_zh_3.0.0_3.0_1615225768619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python word_segmenter = WordSegmenterModel.pretrained("wordseg_ctb9", "zh") .setInputCols(["sentence"]) .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler, word_segmenter]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['从John Snow Labs你好！ ']], ["text"]) result = ws_model.transform(example) ``` ```scala val word_segmenter = WordSegmenterModel.pretrained("wordseg_ctb9", "zh") .setInputCols(Array("sentence")) .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("从John Snow Labs你好！ ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""从John Snow Labs你好！ ""] token_df = nlu.load('zh.segment_words.ctb9').predict(text) token_df ```

## Results ```bash 0 从 1 J 2 o 3 h 4 n 5 S 6 n 7 o 8 w 9 Labs 10 你 11 好 12 ！ Name: token, dtype: object ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_ctb9| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[words_segmented]| |Language:|zh| --- layout: model title: Summarize Clinical Notes in Layman Terms author: John Snow Labs name: summarizer_clinical_laymen date: 2023-05-29 tags: [licensed, en, clinical, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based summarization model that is finetuned with custom dataset by John Snow Labs to avoid using clinical jargon on the summaries. It can generate summaries up to 512 tokens given an input text (max 1024 tokens). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_en_4.4.2_3.0_1685360017257.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_en_4.4.2_3.0_1685360017257.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['This is a clinical note about a 34-year-old woman who is interested in having weight loss surgery. She has been overweight for over 20 years and wants to have more energy and improve her self-image. She has tried many diets and weight loss programs, but has not been successful in keeping the weight off. She has a history of hypertension and shortness of breath, but is not allergic to any medications. She will have an upper endoscopy and will be contacted by a nutritionist and social worker. The plan is to have her weight loss surgery through the gastric bypass, rather than Lap-Band.'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_laymen| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.5 MB| --- layout: model title: Danish asr_xls_r_300m_nst_cv9 TFWav2Vec2ForCTC from chcaa author: John Snow Labs name: asr_xls_r_300m_nst_cv9 date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_300m_nst_cv9` is a Danish model originally trained by chcaa. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_300m_nst_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_300m_nst_cv9_da_4.2.0_3.0_1664103508619.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_300m_nst_cv9_da_4.2.0_3.0_1664103508619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_300m_nst_cv9", "da")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_300m_nst_cv9", "da") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_300m_nst_cv9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|da| |Size:|756.3 MB| --- layout: model title: Sentence Entity Resolver for LOINC (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_loinc_cased date: 2021-12-24 tags: [en, clinical, licensed, entity_resolution, loinc] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted clinical NER entities to LOINC codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It is trained with augmented cased (unlowered) concept names since sbiobert model is cased. ## Predicted Entities `LOINC` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_cased_en_3.3.4_2.4_1640374998947.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_cased_en_3.3.4_2.4_1640374998947.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained('embeddings_clinical','en', 'clinical/models')\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") rad_ner = MedicalNerModel.pretrained("ner_radiology", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") rad_ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ .setWhiteList(['Test']) chunk2doc = Chunk2Doc() \ .setInputCols("ner_chunk") \ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings")\ .setCaseSensitive(True) resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_loinc_cased", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"])\ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline = Pipeline(stages = [ documentAssembler, sentenceDetector, tokenizer, word_embeddings, rad_ner, rad_ner_converter, chunk2doc, sbert_embedder, resolver ]) data = spark.createDataFrame([["""The patient is a 22-year-old female with a history of obesity. She has a BMI of 33.5 kg/m2, aspartate aminotransferase 64, and alanine aminotransferase 126. Her hemoglobin is 8.2%."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical","en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val rad_ner = MedicalNerModel.pretrained("ner_radiology", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val rad_ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") .setWhiteList(Array("Test")) val chunk2doc = Chunk2Doc() .setInputCols("ner_chunk") .setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") .setCaseSensitive(True) val resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_loinc_cased", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, rad_ner, rad_ner_converter, chunk2doc, sbert_embedder, resolver)) val data = Seq("The patient is a 22-year-old female with a history of obesity. She has a BMI of 33.5 kg/m2, aspartate aminotransferase 64, and alanine aminotransferase 126. Her hemoglabin is 8.2%.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.loinc_cased").predict("""The patient is a 22-year-old female with a history of obesity. She has a BMI of 33.5 kg/m2, aspartate aminotransferase 64, and alanine aminotransferase 126. Her hemoglobin is 8.2%.""") ```

## Results ```bash +-------------------------------------+------+-----------+----------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ner_chunk|entity| resolution| all_codes| resolutions| +-------------------------------------+------+-----------+----------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | BMI| Test| LP35925-4|[LP35925-4, 59574-4, BDYCRC, 73964-9, 59574-4,... |[Body mass index (BMI), Body mass index, Body circumference, Body muscle mass, Body mass index (BMI) [Percentile], ... | | aspartate aminotransferase| Test| 14409-7|[14409-7, 1916-6, 16325-3, 16324-6, 43822-6, 308... |[Aspartate aminotransferase, Aspartate aminotransferase/Alanine aminotransferase, Alanine aminotransferase/Aspartate aminotransferase, Alanine aminotransferase, Aspartate aminotransferase [Prese... | | alanine aminotransferase| Test| 16324-6|[16324-6, 16325-3, 14409-7, 1916-6, 59245-1, 30... |[Alanine aminotransferase, Alanine aminotransferase/Aspartate aminotransferase, Aspartate aminotransferase, Aspartate aminotransferase/Alanine aminotransferase, Alanine glyoxylate aminotransfer,... | | hemoglobin| Test| 14775-1|[14775-1, 16931-8, 12710-0, 29220-1, 15082-1, 72... |[Hemoglobin, Hematocrit/Hemoglobin, Hemoglobin pattern, Haptoglobin, Methemoglobin, Oxyhemoglobin, Hemoglobin test status, Verdohemoglobin, Hemoglobin A, Hemoglobin distribution width, Myoglobin,... | +-------------------------------------+------+-----------+----------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_loinc_cased| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[loinc_code]| |Language:|en| |Size:|648.5 MB| |Case sensitive:|true| --- layout: model title: Financial Financial statements Item Binary Classifier author: John Snow Labs name: finclf_financial_statements_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `financial_statements` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `financial_statements` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_financial_statements_item_en_1.0.0_3.2_1660154427604.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_financial_statements_item_en_1.0.0_3.2_1660154427604.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_financial_statements_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[financial_statements]| |[other]| |[other]| |[financial_statements]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_financial_statements_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support financial_statements 0.86 0.96 0.91 1204 other 0.96 0.85 0.90 1254 accuracy - - 0.90 2458 macro-avg 0.91 0.91 0.90 2458 weighted-avg 0.91 0.90 0.90 2458 ``` --- layout: model title: Adverse Drug Events Binary Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_ade_augmented date: 2022-07-27 tags: [clinical, licensed, public_health, ade, classifier, sequence_classification, en] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT-based] (https://github.com/dmis-lab/biobert) classifier that can classify tweets reporting ADEs (Adverse Drug Events). ## Predicted Entities `ADE`, `noADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_ade_augmented_en_4.0.0_3.0_1658905698079.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_ade_augmented_en_4.0.0_3.0_1658905698079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------------------------------------------------------------------------------------+-------+ |text |result | +-----------------------------------------------------------------------------------------------------------------------+-------+ |So glad I am off effexor, so sad it ruined my teeth. tip Please be carefull taking antideppresiva and read about it 1st|[ADE] | |Religare Capital Ranbaxy has been accepting approval for Diovan since 2012 |[noADE]| +-----------------------------------------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_ade_augmented| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## Benchmarking ```bash label precision recall f1-score support ADE 0.9696 0.9595 0.9645 2763 noADE 0.9670 0.9753 0.9712 3366 accuracy - - 0.9682 6129 macro-avg 0.9683 0.9674 0.9678 6129 weighted-avg 0.9682 0.9682 0.9682 6129 ``` --- layout: model title: italian Legal Roberta Embeddings author: John Snow Labs name: roberta_large_italian_legal date: 2023-02-16 tags: [it, italian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-italian-roberta-large` is a Italian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_large_italian_legal_it_4.2.4_3.0_1676557559157.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_large_italian_legal_it_4.2.4_3.0_1676557559157.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_large_italian_legal", "it")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_large_italian_legal", "it") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_large_italian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|it| |Size:|1.3 GB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-italian-roberta-large --- layout: model title: Explain Document Pipeline for Spanish author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, spanish, explain_document_md, pipeline, es] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: es edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_es_3.0.0_3.0_1616431976931.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_es_3.0.0_3.0_1616431976931.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'es') annotations = pipeline.fullAnnotate(""Hola de John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "es") val result = pipeline.fullAnnotate("Hola de John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hola de John Snow Labs! ""] result_df = nlu.load('es.explain.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hola de John Snow Labs! '] | ['Hola de John Snow Labs!'] | ['Hola', 'de', 'John', 'Snow', 'Labs!'] | ['Hola', 'de', 'John', 'Snow', 'Labs!'] | ['PART', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.5123000144958496,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|es| --- layout: model title: Legal Separation Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_separation_agreement date: 2022-11-24 tags: [en, legal, classification, agreement, separation, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_separation_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `separation-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `separation-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_separation_agreement_en_1.0.0_3.0_1669294576564.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_separation_agreement_en_1.0.0_3.0_1669294576564.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[separation-agreement]| |[other]| |[other]| |[separation-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_separation_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.91 0.93 90 separation-agreement 0.82 0.88 0.85 42 accuracy - - 0.90 132 macro-avg 0.88 0.90 0.89 132 weighted-avg 0.90 0.90 0.90 132 ``` --- layout: model title: Pipeline to Mapping RxNorm Codes with Corresponding National Drug Codes (NDC) author: John Snow Labs name: rxnorm_ndc_mapping date: 2022-06-27 tags: [rxnorm, ndc, pipeline, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps RXNORM codes to NDC codes without using any text data. You’ll just feed white space-delimited RXNORM codes and it will return the corresponding two different types of ndc codes which are called `package ndc` and `product ndc`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapping_en_3.5.3_3.0_1656369648141.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapping_en_3.5.3_3.0_1656369648141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rxnorm_ndc_mapping", "en", "clinical/models") result= pipeline.fullAnnotate("1652674 259934") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rxnorm_ndc_mapping", "en", "clinical/models") val result= pipeline.fullAnnotate("1652674 259934") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.rxnorm_to_ndc.pipe").predict("""1652674 259934""") ```

## Results ```bash {'document': ['1652674 259934'], 'package_ndc': ['62135-0625-60', '13349-0010-39'], 'product_ndc': ['46708-0499', '13349-0010'], 'rxnorm_code': ['1652674', '259934']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_ndc_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|4.0 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_ntp0102 TFWav2Vec2ForCTC from ntp0102 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_ntp0102` is a English model originally trained by ntp0102. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102_en_4.2.0_3.0_1664026602923.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102_en_4.2.0_3.0_1664026602923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_ntp0102| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Spanish RobertaForQuestionAnswering (from jamarju) author: John Snow Labs name: roberta_qa_roberta_large_bne_squad_2.0_es_jamarju date: 2022-06-21 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-bne-squad-2.0-es` is a Spanish model originally trained by `jamarju`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_bne_squad_2.0_es_jamarju_es_4.0.0_3.0_1655789415779.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_bne_squad_2.0_es_jamarju_es_4.0.0_3.0_1655789415779.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_bne_squad_2.0_es_jamarju","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_bne_squad_2.0_es_jamarju","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.roberta.large.by_jamarju").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_bne_squad_2.0_es_jamarju| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jamarju/roberta-large-bne-squad-2.0-es - https://github.com/PlanTL-SANIDAD/lm-spanish - https://github.com/ccasimiro88/TranslateAlignRetrieve --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from miamiya) author: John Snow Labs name: roberta_qa_miamiya_base_squad2_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `miamiya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_miamiya_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219308729.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_miamiya_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219308729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_miamiya_base_squad2_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_miamiya_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_miamiya_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/miamiya/roberta-base-squad2-finetuned-squad --- layout: model title: Hungarian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_hungarian_legal date: 2023-02-16 tags: [hu, hungarian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: hu edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-hungarian-roberta-base` is a Hungarian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_hungarian_legal_hu_4.2.4_3.0_1676558480899.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_hungarian_legal_hu_4.2.4_3.0_1676558480899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_hungarian_legal", "hu")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_hungarian_legal", "hu") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_hungarian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hu| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-hungarian-roberta-base --- layout: model title: Word2Vec Embeddings in Sundanese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, su, open_source] task: Embeddings language: su edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_su_3.4.1_3.0_1647459488324.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_su_3.4.1_3.0_1647459488324.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sa_qna","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sa_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sa_qna| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Sachinkelenjaguri/sa_Qna --- layout: model title: Translate English to Austronesian languages Pipeline author: John Snow Labs name: translate_en_map date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, map, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `map` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_map_xx_2.7.0_2.4_1609688461104.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_map_xx_2.7.0_2.4_1609688461104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_map", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_map", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.map').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_map| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_final_784824211 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-final-784824211` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824211_en_4.3.1_3.0_1678134173949.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824211_en_4.3.1_3.0_1678134173949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824211","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824211","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_final_784824211| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-final-784824211 --- layout: model title: Sentence Entity Resolver for billable ICD10-CM HCC codes author: John Snow Labs name: sbiobertresolve_icd10cm_augmented_billable_hcc date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. The load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. It has been augmented with synonyms, four times richer than previous resolver. It also adds support of 7-digit codes with HCC status. ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for `aux_label` parameter in the metadata to get HCC status. The HCC status can be divided to get further information: `billable status`, `hcc status`, and `hcc score`.For example, in the example shared `below the billable status is 1`, `hcc status is 1`, and `hcc score is 8`. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_3.0.4_2.4_1621189647111.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_3.0.4_2.4_1621189647111.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_icd10cm_augmented_billable_hcc``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. ```PROBLEM``` set in ```.setWhiteList()```.

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented_billable_hcc| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: Oncology Pipeline for Therapies author: John Snow Labs name: oncology_therapy_pipeline date: 2023-03-29 tags: [licensed, pipeline, oncology, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition and Assertion Status models to extract information from oncology texts. This pipeline focuses on entities related to therapies. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_therapy_pipeline_en_4.3.2_3.2_1680123025997.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_therapy_pipeline_en_4.3.2_3.2_1680123025997.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_therapy_pipeline", "en", "clinical/models") text = '''The patient underwent a mastectomy two years ago. She is currently receiving her second cycle of adriamycin and cyclophosphamide, and is in good overall condition.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_therapy_pipeline", "en", "clinical/models") val text = "The patient underwent a mastectomy two years ago. She is currently receiving her second cycle of adriamycin and cyclophosphamide, and is in good overall condition." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_therpay.pipeline").predict("""The patient underwent a mastectomy two years ago. She is currently receiving her second cycle of adriamycin and cyclophosphamide, and is in good overall condition.""") ```

## Results ```bash " ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-----------------|:---------------| | mastectomy | Cancer_Surgery | | second cycle | Cycle_Number | | adriamycin | Chemotherapy | | cyclophosphamide | Chemotherapy | ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-----------------|:---------------| | mastectomy | Cancer_Surgery | | second cycle | Cycle_Number | | adriamycin | Chemotherapy | | cyclophosphamide | Chemotherapy | ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-----------------|:---------------| | mastectomy | Cancer_Surgery | | second cycle | Cycle_Number | | adriamycin | Cancer_Therapy | | cyclophosphamide | Cancer_Therapy | ******************** ner_oncology_unspecific_posology_wip results ******************** | chunk | ner_label | |:-----------------|:---------------------| | mastectomy | Cancer_Therapy | | second cycle | Posology_Information | | adriamycin | Cancer_Therapy | | cyclophosphamide | Cancer_Therapy | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:-----------------|:---------------|:------------| | mastectomy | Cancer_Surgery | Past | | adriamycin | Chemotherapy | Present | | cyclophosphamide | Chemotherapy | Present | ******************** assertion_oncology_treatment_binary_wip results ******************** | chunk | ner_label | assertion | |:-----------------|:---------------|:----------------| | mastectomy | Cancer_Surgery | Present_Or_Past | | adriamycin | Chemotherapy | Present_Or_Past | | cyclophosphamide | Chemotherapy | Present_Or_Past | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_therapy_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - AssertionDLModel --- layout: model title: Arabic ElectraForQuestionAnswering model (from wissamantoun) author: John Snow Labs name: electra_qa_ara_base_artydiqa date: 2022-06-22 tags: [ar, open_source, electra, question_answering] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `araelectra-base-artydiqa` is a Arabic model originally trained by `wissamantoun`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_ara_base_artydiqa_ar_4.0.0_3.0_1655920272375.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_ara_base_artydiqa_ar_4.0.0_3.0_1655920272375.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_ara_base_artydiqa","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_ara_base_artydiqa","ar") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.tydiqa.electra.base").predict("""ما هو اسمي؟|||"اسمي كلارا وأنا أعيش في بيركلي.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_ara_base_artydiqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ar| |Size:|504.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/wissamantoun/araelectra-base-artydiqa --- layout: model title: Explain Document Pipeline for Italian author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, italian, explain_document_md, pipeline, it] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: it edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_it_3.0.0_3.0_1616430477970.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_it_3.0.0_3.0_1616430477970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'it') annotations = pipeline.fullAnnotate(""Ciao da John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "it") val result = pipeline.fullAnnotate("Ciao da John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Ciao da John Snow Labs! ""] result_df = nlu.load('it.explain.document').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Ciao da John Snow Labs! '] | ['Ciao da John Snow Labs!'] | ['Ciao', 'da', 'John', 'Snow', 'Labs!'] | ['Ciao', 'da', 'John', 'Snow', 'Labs!'] | ['VERB', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[-0.146050006151199,.,...]] | ['O', 'O', 'I-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|it| --- layout: model title: Chinese BertForMaskedLM Large Cased model (from genggui001) author: John Snow Labs name: bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_wwm_large_ext_fix_mlm` is a Chinese model originally trained by `genggui001`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm_zh_4.2.4_3.0_1670326139931.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm_zh_4.2.4_3.0_1670326139931.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_wwm_large_ext_fix_mlm| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/genggui001/chinese_roberta_wwm_large_ext_fix_mlm - https://github.com/ymcui/Chinese-BERT-wwm/issues/98 - https://github.com/genggui001/chinese_roberta_wwm_large_ext_fix_mlm --- layout: model title: English DistilBertForQuestionAnswering model (from machine2049) Duorc author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_duorc_ date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-duorc_distilbert` is a English model originally trained by `machine2049`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_duorc__en_4.0.0_3.0_1654723876220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_duorc__en_4.0.0_3.0_1654723876220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_duorc_","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_duorc_","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_uncased.by_machine2049").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_duorc_| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/machine2049/distilbert-base-uncased-finetuned-duorc_distilbert --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from kaouther) author: John Snow Labs name: distilbert_qa_kaouther_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `kaouther`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_kaouther_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771677866.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_kaouther_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771677866.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kaouther_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kaouther_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_kaouther_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kaouther/distilbert-base-uncased-finetuned-squad --- layout: model title: French CamemBert Embeddings (from gulabpatel) author: John Snow Labs name: camembert_embeddings_new_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `new-dummy-model` is a French model orginally trained by `gulabpatel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_new_generic_model_fr_3.4.4_3.0_1653991782298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_new_generic_model_fr_3.4.4_3.0_1653991782298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_new_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/gulabpatel/new-dummy-model --- layout: model title: German Financial Bert Word Embeddings author: John Snow Labs name: bert_sentence_embeddings_financial date: 2022-05-04 tags: [bert, embeddings, de, open_source, financial] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Although in the name of the model you will see the word `sentence`, this is a Word Embeddings Model. Financial Pretrained BERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `german-financial-statements-bert` is a German model orginally trained by `fabianrausch`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sentence_embeddings_financial_de_3.4.2_3.0_1651678415089.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sentence_embeddings_financial_de_3.4.2_3.0_1651678415089.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sentence_embeddings_financial| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|409.6 MB| |Case sensitive:|true| --- layout: model title: English asr_wav2vec2_base_timit_ali_hasan_colab_EX2 TFWav2Vec2ForCTC from ali221000262 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_ali_hasan_colab_EX2` is a English model originally trained by ali221000262. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2_en_4.2.0_3.0_1664038560611.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2_en_4.2.0_3.0_1664038560611.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_ali_hasan_colab_EX2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Named Entity Recognition (NER) Model in Norwegian (Norne 6B 100) author: John Snow Labs name: norne_6B_100 date: 2020-05-06 task: Named Entity Recognition language: "no" edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [ner, nn, nb, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Norne is a Named Entity Recognition (or NER) model of Norvegian, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. Norne 6B 100 is trained with GloVe 6B 100 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Derived-`DRV`, Product-`PROD`, Geo-political Entities Location-`GPE_LOC`, Geo-political Entities Organization-`GPE-ORG`, Event-`EVT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_NO/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_NO.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/norne_6B_300_no_2.5.0_2.4_1588781290264.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/norne_6B_300_no_2.5.0_2.4_1588781290264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained('glove_6B_100') \ .setInputCols(['document', 'token']) \ .setOutputCol('embeddings') ner_model = NerDLModel.pretrained("norne_6B_100", "no") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([['William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. [ 9] Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella.']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_6B_100") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("norne_6B_100", "no") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. [ 9] Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella."""] ner_df = nlu.load('no.ner.norne.glove.6B_100').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates III |PER | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |Seattle |GPE_LOC | |Washington |GPE_LOC | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |GPE_LOC | |New Mexico |GPE_LOC | |Gates |PER | |Gates |PER | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|ORG | |Melinda Gates |PER | |Ray Ozzie |PER | |Craig Mundie |PER | |Han |PER | |Microsoft |ORG | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|norne_6B_100| |Type:|ner| |Compatibility:| Spark NLP 2.5.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|no| |Case sensitive:|false| {:.h2_title} ## Data Source The detailed information can be found from [https://www.aclweb.org/anthology/2020.lrec-1.559.pdf](https://www.aclweb.org/anthology/2020.lrec-1.559.pdf) --- layout: model title: Lemmatizer (Lithuanian, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, lt] task: Lemmatization language: lt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Lithuanian Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_lt_3.4.1_3.0_1646316598333.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_lt_3.4.1_3.0_1646316598333.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+ |result | +------------------------------+ |[Jūs, nebūti, geras, už, mane]| +------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|lt| |Size:|2.6 MB| --- layout: model title: Spanish BertForTokenClassification Cased model (from luch0247) author: John Snow Labs name: bert_token_classifier_autotrain_lucy_alicorp_1356152290 date: 2022-11-30 tags: [es, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-Lucy-Alicorp-1356152290` is a Spanish model originally trained by `luch0247`. ## Predicted Entities `C`, `NM`, `VRB`, `CR`, `QT`, `DB` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_lucy_alicorp_1356152290_es_4.2.4_3.0_1669814335691.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_lucy_alicorp_1356152290_es_4.2.4_3.0_1669814335691.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_lucy_alicorp_1356152290","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_lucy_alicorp_1356152290","es") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autotrain_lucy_alicorp_1356152290| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|410.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/luch0247/autotrain-Lucy-Alicorp-1356152290 --- layout: model title: Fast Neural Machine Translation Model from English to Twi author: John Snow Labs name: opus_mt_en_tw date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tw, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tw_xx_2.7.0_2.4_1609169865968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tw_xx_2.7.0_2.4_1609169865968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tw| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English XLMRobertaForTokenClassification Base Cased model (from AI4Sec) author: John Snow Labs name: xlmroberta_ner_cyner_base date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cyner-xlm-roberta-base` is a English model originally trained by `AI4Sec`. ## Predicted Entities `Vulnerability`, `Malware`, `System`, `Organization`, `Indicator` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cyner_base_en_4.1.0_3.0_1660422140565.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cyner_base_en_4.1.0_3.0_1660422140565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cyner_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cyner_base","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_cyner_base| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|780.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AI4Sec/cyner-xlm-roberta-base --- layout: model title: English asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves TFWav2Vec2ForCTC from tonyalves author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves` is a English model originally trained by tonyalves. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_en_4.2.0_3.0_1664109299079.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_en_4.2.0_3.0_1664109299079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_base_finetuned_scrambled_squad_15 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-15` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_15_en_4.3.0_3.0_1674216826395.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_15_en_4.3.0_3.0_1674216826395.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_15","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_15","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_scrambled_squad_15| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-15 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-42` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42_en_4.3.0_3.0_1674213511887.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42_en_4.3.0_3.0_1674213511887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_42| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|447.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-42 --- layout: model title: Word Segmenter for Chinese author: John Snow Labs name: wordseg_pku date: 2021-03-09 tags: [word_segmentation, open_source, chinese, wordseg_pku, zh] task: Word Segmentation language: zh edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [WordSegmenterModel-WSM](https://en.wikipedia.org/wiki/Text_segmentation) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know a priori which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/chinese/word_segmentation/words_segmenter_demo.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_pku_zh_3.0.0_3.0_1615292332841.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_pku_zh_3.0.0_3.0_1615292332841.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python word_segmenter = WordSegmenterModel.pretrained("wordseg_pku", "zh") .setInputCols(["sentence"]) .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler, word_segmenter]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['从John Snow Labs你好！ ']], ["text"]) result = ws_model.transform(example) ``` ```scala val word_segmenter = WordSegmenterModel.pretrained("wordseg_pku", "zh") .setInputCols(Array("sentence")) .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("从John Snow Labs你好！ ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""从John Snow Labs你好！ ""] token_df = nlu.load('zh.segment_words.pku').predict(text) token_df ```

## Results ```bash 0 从 1 Jo 2 hn 3 Sn 4 ow 5 La 6 bs 7 你 8 好 9 ！ Name: token, dtype: object ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_pku| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[words_segmented]| |Language:|zh| --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-2_H-512_A-8_cord19-200616_squad2` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2_en_4.0.0_3.0_1654185223441.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2_en_4.0.0_3.0_1654185223441.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_cord19.bert.uncased_2l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_2_H_512_A_8_cord19_200616_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|83.4 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-2_H-512_A-8_cord19-200616_squad2 --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from it5) author: John Snow Labs name: t5_it5_efficient_small_el32_informal_to_formal date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-el32-informal-to-formal` is a Italian model originally trained by `it5`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_informal_to_formal_it_4.3.0_3.0_1675103414416.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_informal_to_formal_it_4.3.0_3.0_1675103414416.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_informal_to_formal","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_informal_to_formal","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_el32_informal_to_formal| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|593.5 MB| ## References - https://huggingface.co/it5/it5-efficient-small-el32-informal-to-formal - https://github.com/stefan-it - https://arxiv.org/abs/2203.03759 - https://gsarti.com - https://malvinanissim.github.io - https://arxiv.org/abs/2109.10686 - https://github.com/gsarti/it5 - https://paperswithcode.com/sota?task=Informal-to-formal+Style+Transfer&dataset=XFORMAL+%28Italian+Subset%29 --- layout: model title: Legal Powers Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_powers_bert date: 2023-03-05 tags: [en, legal, classification, clauses, powers, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Powers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Powers`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_powers_bert_en_1.0.0_3.0_1678050727123.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_powers_bert_en_1.0.0_3.0_1678050727123.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_powers_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Powers]| |[Other]| |[Other]| |[Powers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_powers_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.3 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.90 0.95 0.92 19 Powers 0.90 0.82 0.86 11 accuracy - - 0.90 30 macro-avg 0.90 0.88 0.89 30 weighted-avg 0.90 0.90 0.90 30 ``` --- layout: model title: Korean BertForMaskedLM Base Cased model (from kykim) author: John Snow Labs name: bert_embeddings_kor_base date: 2022-12-02 tags: [ko, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ko edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-kor-base` is a Korean model originally trained by `kykim`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_kor_base_ko_4.2.4_3.0_1670019610924.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_kor_base_ko_4.2.4_3.0_1670019610924.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kor_base","ko") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kor_base","ko") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_kor_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ko| |Size:|443.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/kykim/bert-kor-base - https://github.com/kiyoungkim1/LM-kor --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from moghis) author: John Snow Labs name: xlmroberta_ner_base_finetuned_panx_de_data date: 2022-08-14 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de-data` is a German model originally trained by `moghis`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_panx_de_data_de_4.1.0_3.0_1660438168971.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_panx_de_data_de_4.1.0_3.0_1660438168971.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_panx_de_data","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_panx_de_data","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_panx_de_data| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/moghis/xlm-roberta-base-finetuned-panx-de-data --- layout: model title: French CamemBert Embeddings (from kaushikacharya) author: John Snow Labs name: camembert_embeddings_kaushikacharya_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `kaushikacharya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_kaushikacharya_generic_model_fr_3.4.4_3.0_1653989224371.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_kaushikacharya_generic_model_fr_3.4.4_3.0_1653989224371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_kaushikacharya_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/kaushikacharya/dummy-model --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10_en_4.0.0_3.0_1655731588944.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10_en_4.0.0_3.0_1655731588944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|416.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-10 --- layout: model title: Pipeline to Resolve Medication Codes author: John Snow Labs name: medication_resolver_pipeline date: 2023-04-10 tags: [resolver, snomed, umls, rxnorm, ndc, ade, en, licensed, pipeline] task: Entity Resolution language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained resolver pipeline to extract medications and resolve their adverse reactions (ADE), RxNorm, UMLS, NDC, SNOMED CT codes, and action/treatments in clinical text. Action/treatments are available for branded medication, and SNOMED codes are available for non-branded medication. This pipeline can be used as Lightpipeline (with `annotate/fullAnnotate`). You can use `medication_resolver_transform_pipeline` for Spark transform. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/medication_resolver_pipeline_en_4.3.2_3.0_1681151954032.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/medication_resolver_pipeline_en_4.3.2_3.0_1681151954032.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline med_resolver_pipeline = PretrainedPipeline("medication_resolver_pipeline", "en", "clinical/models") text = """The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""" result = med_resolver_pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val med_resolver_pipeline = new PretrainedPipeline("medication_resolver_pipeline", "en", "clinical/models") val result = med_resolver_pipeline.fullAnnotate("""The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.medication").predict("""The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""") ```

## Results ```bash | | chunks | entities | ADE | RxNorm | Action | Treatment | UMLS | SNOMED_CT | NDC_Product | NDC_Package | |---:|:-----------------------------|:-----------|:----------------------------|---------:|:---------------------------|:-------------------------------------------|:---------|:------------|:--------------|:--------------| | 0 | Amlodopine Vallarta 10-320mg | DRUG | Gynaecomastia | 722131 | NONE | NONE | C1949334 | 425838008 | 00093-7693 | 00093-7693-56 | | 1 | Eviplera | DRUG | Anxiety | 217010 | Inhibitory Bone Resorption | Osteoporosis | C0720318 | NONE | NONE | NONE | | 2 | Lescol 40 MG | DRUG | NONE | 103919 | Hypocholesterolemic | Heterozygous Familial Hypercholesterolemia | C0353573 | NONE | 00078-0234 | 00078-0234-05 | | 3 | Everolimus 1.5 mg tablet | DRUG | Acute myocardial infarction | 2056895 | NONE | NONE | C4723581 | NONE | 00054-0604 | 00054-0604-21 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|medication_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.2 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - TextMatcherModel - ChunkMergeModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger - ResolverMerger - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - Finisher --- layout: model title: Fast Neural Machine Translation Model from Arabic to Esperanto author: John Snow Labs name: opus_mt_ar_eo date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, eo, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: eo {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_eo_xx_3.1.0_2.4_1622554268155.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_eo_xx_3.1.0_2.4_1622554268155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_eo| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Emotional Stressor Classifier (BERT) author: John Snow Labs name: bert_sequence_classifier_stressor date: 2022-07-27 tags: [stressor, public_health, en, licensed, sequence_classification] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [bioBERT](https://nlp.johnsnowlabs.com/2022/07/18/biobert_pubmed_base_cased_v1.2_en_3_0.html) based classifier that can classify source of emotional stress in text. ## Predicted Entities `Family_Issues`, `Financial_Problem`, `Health_Fatigue_or_Physical Pain`, `Other`, `School`, `Work`, `Social_Relationships` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_STRESS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_stressor_en_4.0.0_3.0_1658923809554.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_stressor_en_4.0.0_3.0_1658923809554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------------------------------------------+-----------------------------------+ |text |class | +------------------------------------------------------------------+-----------------------------------+ |All the panic about the global pandemic has been stressing me out!|[Health, Fatigue, or Physical Pain]| +------------------------------------------------------------------+-----------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_stressor| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## Benchmarking ```bash label precision recall f1-score support Family Issues 0.80 0.87 0.84 161 Financial Problem 0.87 0.83 0.85 126 Health, Fatigue, or Physical Pain 0.75 0.81 0.78 168 Other 0.82 0.80 0.81 384 School 0.89 0.91 0.90 127 Social Relationships 0.83 0.71 0.76 133 Work 0.87 0.89 0.88 271 accuracy - - 0.83 1370 macro-avg 0.83 0.83 0.83 1370 weighted-avg 0.83 0.83 0.83 1370 ``` --- layout: model title: Question classification of open-domain and fact-based questions Pipeline - TREC6 author: John Snow Labs name: classifierdl_use_trec6_pipeline date: 2021-01-08 task: [Text Classification, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [classifier, text_classification, en, open_source, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify open-domain, fact-based questions into one of the following broad semantic categories: Abbreviation, Description, Entities, Human Beings, Locations or Numeric Values. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_EN_TREC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_EN_TREC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec6_pipeline_en_2.7.1_2.4_1610119335714.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec6_pipeline_en_2.7.1_2.4_1610119335714.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("classifierdl_use_trec6_pipeline", lang = "en") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("classifierdl_use_trec6_pipeline", lang = "en") ```

## Results ```bash +------------------------------------------------------------------------------------------------+------------+ |document |class | +------------------------------------------------------------------------------------------------+------------+ |When did the construction of stone circles begin in the UK? | NUM | +------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_trec6_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Language:|en| --- layout: model title: Translate English to Luo (Kenya and Tanzania) Pipeline author: John Snow Labs name: translate_en_luo date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, luo, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `luo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_luo_xx_2.7.0_2.4_1609689219424.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_luo_xx_2.7.0_2.4_1609689219424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_luo", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_luo", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.luo').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_luo| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xlsr_53_gpt TFWav2Vec2ForCTC from voidful author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_gpt date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_gpt` is a English model originally trained by voidful. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_gpt_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_gpt_en_4.2.0_3.0_1664095296861.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_gpt_en_4.2.0_3.0_1664095296861.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_gpt', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_gpt", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_gpt| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.3 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Abkhazian asr_hf_challenge_test TFWav2Vec2ForCTC from Iskaj author: John Snow Labs name: asr_hf_challenge_test date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_hf_challenge_test` is a Abkhazian model originally trained by Iskaj. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_hf_challenge_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_hf_challenge_test_ab_4.2.0_3.0_1664021278980.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_hf_challenge_test_ab_4.2.0_3.0_1664021278980.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_hf_challenge_test", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_hf_challenge_test", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_hf_challenge_test| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|446.6 KB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from arjunth2001) author: John Snow Labs name: roberta_qa_priv_qna date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `priv_qna` is a English model originally trained by `arjunth2001`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_priv_qna_en_4.3.0_3.0_1674211774365.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_priv_qna_en_4.3.0_3.0_1674211774365.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_priv_qna","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_priv_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_priv_qna| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/arjunth2001/priv_qna --- layout: model title: English BertForQuestionAnswering Small Cased model (from motiondew) author: John Snow Labs name: bert_qa_sd2_small date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-sd2-small` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sd2_small_en_4.0.0_3.0_1657188173526.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sd2_small_en_4.0.0_3.0_1657188173526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sd2_small","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_sd2_small","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sd2_small| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-sd2-small --- layout: model title: Translate Niger-Kordofanian languages to English Pipeline author: John Snow Labs name: translate_nic_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, nic, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `nic` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_nic_en_xx_2.7.0_2.4_1609699199544.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_nic_en_xx_2.7.0_2.4_1609699199544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_nic_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_nic_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.nic.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_nic_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Extract Biomarkers and Their Results author: John Snow Labs name: ner_oncology_biomarker_healthcare_pipeline date: 2023-03-08 tags: [licensed, clinical, oncology, en, ner, biomarker] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_biomarker_healthcare](https://nlp.johnsnowlabs.com/2023/01/11/ner_oncology_biomarker_healthcare_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_healthcare_pipeline_en_4.3.0_3.2_1678269721297.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_healthcare_pipeline_en_4.3.0_3.2_1678269721297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_biomarker_healthcare_pipeline", "en", "clinical/models") text = '''he results of immunohistochemical examination showed that she tested negative for CK7, synaptophysin (Syn), chromogranin A (CgA), Muc5AC, human epidermal growth factor receptor-2 (HER2), and Muc6; positive for CK20, Muc1, Muc2, E-cadherin, and p53; the Ki-67 index was about 87%.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_biomarker_healthcare_pipeline", "en", "clinical/models") val text = "he results of immunohistochemical examination showed that she tested negative for CK7, synaptophysin (Syn), chromogranin A (CgA), Muc5AC, human epidermal growth factor receptor-2 (HER2), and Muc6; positive for CK20, Muc1, Muc2, E-cadherin, and p53; the Ki-67 index was about 87%." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:-----------------------------------------|--------:|------:|:-----------------|-------------:| | 0 | negative | 69 | 76 | Biomarker_Result | 1 | | 1 | CK7 | 82 | 84 | Biomarker | 1 | | 2 | synaptophysin | 87 | 99 | Biomarker | 1 | | 3 | Syn | 102 | 104 | Biomarker | 0.9999 | | 4 | chromogranin A | 108 | 121 | Biomarker | 0.99855 | | 5 | CgA | 124 | 126 | Biomarker | 1 | | 6 | Muc5AC | 130 | 135 | Biomarker | 0.9999 | | 7 | human epidermal growth factor receptor-2 | 138 | 177 | Biomarker | 0.99994 | | 8 | HER2 | 180 | 183 | Biomarker | 1 | | 9 | Muc6 | 191 | 194 | Biomarker | 1 | | 10 | positive | 197 | 204 | Biomarker_Result | 0.9997 | | 11 | CK20 | 210 | 213 | Biomarker | 1 | | 12 | Muc1 | 216 | 219 | Biomarker | 1 | | 13 | Muc2 | 222 | 225 | Biomarker | 1 | | 14 | E-cadherin | 228 | 237 | Biomarker | 0.9997 | | 15 | p53 | 244 | 246 | Biomarker | 1 | | 16 | Ki-67 index | 253 | 263 | Biomarker | 0.99865 | | 17 | 87% | 275 | 277 | Biomarker_Result | 0.828 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_biomarker_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|533.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from phiyodr) author: John Snow Labs name: roberta_qa_large_finetuned_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-finetuned-squad2` is a English model originally trained by `phiyodr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_finetuned_squad2_en_4.2.4_3.0_1669987751520.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_finetuned_squad2_en_4.2.4_3.0_1669987751520.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_finetuned_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_finetuned_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/phiyodr/roberta-large-finetuned-squad2 - https://rajpurkar.github.io/SQuAD-explorer/ - https://arxiv.org/abs/1907.11692 - https://arxiv.org/abs/1806.03822 - https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from sasuke) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_squad1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad1` is a English model originally trained by `sasuke`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad1_en_4.3.0_3.0_1672773467636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad1_en_4.3.0_3.0_1672773467636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_squad1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sasuke/distilbert-base-uncased-finetuned-squad1 --- layout: model title: Legal Warrant Agreement Document Binary Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_warrant_agreement_bert date: 2022-12-18 tags: [en, legal, classification, licensed, document, bert, warrant, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_warrant_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `warrant-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `warrant-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_warrant_agreement_bert_en_1.0.0_3.0_1671393844438.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_warrant_agreement_bert_en_1.0.0_3.0_1671393844438.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_warrant_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[warrant-agreement]| |[other]| |[other]| |[warrant-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_warrant_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.99 0.98 204 warrant-agreement 0.96 0.95 0.96 83 accuracy - - 0.98 287 macro-avg 0.97 0.97 0.97 287 weighted-avg 0.98 0.98 0.98 287 ``` --- layout: model title: Pipeline to Detect Anatomical Regions (MedicalBertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_anatomy_pipeline date: 2023-03-20 tags: [anatomy, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_anatomy](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_anatomy_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_pipeline_en_4.3.0_3.2_1679306174114.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_pipeline_en_4.3.0_3.2_1679306174114.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_anatomy_pipeline", "en", "clinical/models") text = '''This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist. Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_anatomy_pipeline", "en", "clinical/models") val text = "This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist. Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.anatomy_pipeline").predict("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist. Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:-----------------------|-------------:| | 0 | great | 320 | 324 | Multi-tissue_structure | 0.693343 | | 1 | toe | 326 | 328 | Multi-tissue_structure | 0.378996 | | 2 | skin | 374 | 377 | Organ | 0.946453 | | 3 | conjunctivae | 554 | 565 | Multi-tissue_structure | 0.929193 | | 4 | Extraocular | 574 | 584 | Multi-tissue_structure | 0.858331 | | 5 | muscles | 586 | 592 | Organ | 0.670788 | | 6 | Nares | 613 | 617 | Multi-tissue_structure | 0.573931 | | 7 | turbinates | 659 | 668 | Multi-tissue_structure | 0.947797 | | 8 | Oropharynx | 683 | 692 | Multi-tissue_structure | 0.458301 | | 9 | Mucous membranes | 716 | 731 | Tissue | 0.811466 | | 10 | Neck | 744 | 747 | Organism_subdivision | 0.879527 | | 11 | bowel | 802 | 806 | Organ | 0.919502 | | 12 | great | 875 | 879 | Multi-tissue_structure | 0.701514 | | 13 | toe | 881 | 883 | Multi-tissue_structure | 0.264513 | | 14 | skin | 933 | 936 | Organ | 0.925361 | | 15 | toenails | 943 | 950 | Organism_subdivision | 0.674937 | | 16 | foot | 999 | 1002 | Organism_subdivision | 0.544587 | | 17 | great | 1017 | 1021 | Multi-tissue_structure | 0.818323 | | 18 | toe | 1023 | 1025 | Organism_subdivision | 0.341098 | | 19 | toenails | 1031 | 1038 | Organism_subdivision | 0.75016 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_anatomy_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Luo (Kenya and Tanzania) XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_luo_finetuned_ner date: 2022-08-01 tags: [luo, open_source, xlm_roberta, ner] task: Named Entity Recognition language: luo edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-luo-finetuned-ner-luo` is a Luo (Kenya and Tanzania) model originally trained by `mbeukman`. ## Predicted Entities `PER`, `DATE`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_luo_finetuned_ner_luo_4.1.0_3.0_1659354450889.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_luo_finetuned_ner_luo_4.1.0_3.0_1659354450889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_luo_finetuned_ner","luo") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_luo_finetuned_ner","luo") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_luo_finetuned_ner| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|luo| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-luo-finetuned-ner-luo - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: English image_classifier_vit_cifar10 ViTForImageClassification from alfredcs author: John Snow Labs name: image_classifier_vit_cifar10 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_cifar10` is a English model originally trained by alfredcs. ## Predicted Entities `deer`, `bird`, `dog`, `horse`, `automobile`, `truck`, `frog`, `ship`, `airplane`, `cat` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_cifar10_en_4.1.0_3.0_1660167465918.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_cifar10_en_4.1.0_3.0_1660167465918.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_cifar10", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_cifar10", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_cifar10| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from ncduy) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_finetuned_test date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-finetuned-squad-test` is a English model originally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_finetuned_test_en_4.3.0_3.0_1672766662366.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_finetuned_test_en_4.3.0_3.0_1672766662366.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_finetuned_test","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_finetuned_test","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_finetuned_test| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/distilbert-base-cased-distilled-squad-finetuned-squad-test --- layout: model title: Indonesian Part of Speech Tagger (from w11wo) author: John Snow Labs name: roberta_pos_indonesian_roberta_base_posp_tagger date: 2022-05-03 tags: [roberta, pos, part_of_speech, id, open_source] task: Part of Speech Tagging language: id edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indonesian-roberta-base-posp-tagger` is a Indonesian model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_pos_indonesian_roberta_base_posp_tagger_id_3.4.2_3.0_1651596272433.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_pos_indonesian_roberta_base_posp_tagger_id_3.4.2_3.0_1651596272433.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_indonesian_roberta_base_posp_tagger| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|id| |Size:|466.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/w11wo/indonesian-roberta-base-posp-tagger - https://arxiv.org/abs/1907.11692 - https://hf.co/flax-community/indonesian-roberta-base - https://hf.co/datasets/indonlu - https://w11wo.github.io/ --- layout: model title: Multilingual Representations for Indian Languages (MuRIL) - BERT Sentence Embedding pre-trained on 17 Indian languages author: John Snow Labs name: sent_bert_muril date: 2021-09-01 tags: [xx, open_source, sentence_embeddings, muril, indian_languages] task: Embeddings language: xx edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture pretrained from scratch using the Wikipedia, Common Crawl, PMINDIA and Dakshina corpora for the following 17 Indian languages: `Assamese`, `Bengali` , `English` , `Gujarati` , `Hindi` , `Kannada` , `Kashmiri` , `Malayalam` , `Marathi` , `Nepali` , `Oriya` , `Punjabi` , `Sanskrit` , `Sindhi` , `Tamil` , `Telugu` , `Urdu` The MuRIL model is pre-trained on monolingual segments as well as parallel segments as detailed below : - Monolingual Data : Publicly available corpora from Wikipedia and Common Crawl for 17 Indian languages. - Parallel Data : There are two types of parallel data : - Translated Data : Translations of the above monolingual corpora obtained using the Google NMT pipeline. Translated segment pairs fed as input. Also, Publicly available PMINDIA corpus was used. - Transliterated Data : Transliterations of Wikipedia obtained using the IndicTrans library. Transliterated segment pairs fed as input. Also, Publicly available Dakshina dataset was used. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_muril_xx_3.2.0_3.0_1630467991919.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_muril_xx_3.2.0_3.0_1630467991919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_muril", "xx") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_muril", "xx") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.muril').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_muril| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|xx| |Case sensitive:|false| ## Data Source [1]: Jacob Devlin, Ming-Wei Chang, Kenton Lee, Kristina Toutanova. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. arXiv preprint arXiv:1810.04805, 2018. [2]: [Wikipedia](https://www.tensorflow.org/datasets/catalog/wikipedia) [3]: [Common Crawl](http://commoncrawl.org/the-data/) [4]: [PMINDIA](http://lotus.kuee.kyoto-u.ac.jp/WAT/indic-multilingual/index.html) [5]: [Dakshina](https://github.com/google-research-datasets/dakshina) The model is imported from: https://tfhub.dev/google/MuRIL/1 --- layout: model title: Sentence Detection in Punjabi Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [pa, open_source, sentence_detection] task: Sentence Detection language: pa edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_pa_3.2.0_3.0_1630320087911.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_pa_3.2.0_3.0_1630320087911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "pa") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਦੇ ਪੈਰਾਗ੍ਰਾਫਾਂ ਦੇ ਇੱਕ ਮਹਾਨ ਸਰੋਤ ਦੀ ਭਾਲ ਕਰ ਰਹੇ ਹੋ? ਤੁਸੀਂ ਸਹੀ ਜਗ੍ਹਾ ਤੇ ਆਏ ਹੋ. ਇੱਕ ਤਾਜ਼ਾ ਅਧਿਐਨ ਅਨੁਸਾਰ ਅੱਜ ਦੇ ਨੌਜਵਾਨਾਂ ਵਿੱਚ ਪੜ੍ਹਨ ਦੀ ਆਦਤ ਤੇਜ਼ੀ ਨਾਲ ਘਟ ਰਹੀ ਹੈ। ਉਹ ਕੁਝ ਸਕਿੰਟਾਂ ਤੋਂ ਵੱਧ ਸਮੇਂ ਲਈ ਦਿੱਤੇ ਗਏ ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਵਾਲੇ ਪੈਰੇ 'ਤੇ ਧਿਆਨ ਨਹੀਂ ਦੇ ਸਕਦੇ! ਨਾਲ ਹੀ, ਪੜ੍ਹਨਾ ਸਾਰੀਆਂ ਪ੍ਰਤੀਯੋਗੀ ਪ੍ਰੀਖਿਆਵਾਂ ਦਾ ਇੱਕ ਅਨਿੱਖੜਵਾਂ ਅੰਗ ਸੀ ਅਤੇ ਹੈ. ਇਸ ਲਈ, ਤੁਸੀਂ ਆਪਣੇ ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਨੂੰ ਕਿਵੇਂ ਸੁਧਾਰਦੇ ਹੋ? ਇਸ ਪ੍ਰਸ਼ਨ ਦਾ ਉੱਤਰ ਅਸਲ ਵਿੱਚ ਇੱਕ ਹੋਰ ਪ੍ਰਸ਼ਨ ਹੈ: ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਦੀ ਵਰਤੋਂ ਕੀ ਹੈ? ਪੜ੍ਹਨ ਦਾ ਮੁੱਖ ਉਦੇਸ਼ 'ਅਰਥ ਬਣਾਉਣਾ' ਹੈ.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "pa") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਦੇ ਪੈਰਾਗ੍ਰਾਫਾਂ ਦੇ ਇੱਕ ਮਹਾਨ ਸਰੋਤ ਦੀ ਭਾਲ ਕਰ ਰਹੇ ਹੋ? ਤੁਸੀਂ ਸਹੀ ਜਗ੍ਹਾ ਤੇ ਆਏ ਹੋ. ਇੱਕ ਤਾਜ਼ਾ ਅਧਿਐਨ ਅਨੁਸਾਰ ਅੱਜ ਦੇ ਨੌਜਵਾਨਾਂ ਵਿੱਚ ਪੜ੍ਹਨ ਦੀ ਆਦਤ ਤੇਜ਼ੀ ਨਾਲ ਘਟ ਰਹੀ ਹੈ। ਉਹ ਕੁਝ ਸਕਿੰਟਾਂ ਤੋਂ ਵੱਧ ਸਮੇਂ ਲਈ ਦਿੱਤੇ ਗਏ ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਵਾਲੇ ਪੈਰੇ 'ਤੇ ਧਿਆਨ ਨਹੀਂ ਦੇ ਸਕਦੇ! ਨਾਲ ਹੀ, ਪੜ੍ਹਨਾ ਸਾਰੀਆਂ ਪ੍ਰਤੀਯੋਗੀ ਪ੍ਰੀਖਿਆਵਾਂ ਦਾ ਇੱਕ ਅਨਿੱਖੜਵਾਂ ਅੰਗ ਸੀ ਅਤੇ ਹੈ. ਇਸ ਲਈ, ਤੁਸੀਂ ਆਪਣੇ ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਨੂੰ ਕਿਵੇਂ ਸੁਧਾਰਦੇ ਹੋ? ਇਸ ਪ੍ਰਸ਼ਨ ਦਾ ਉੱਤਰ ਅਸਲ ਵਿੱਚ ਇੱਕ ਹੋਰ ਪ੍ਰਸ਼ਨ ਹੈ: ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਦੀ ਵਰਤੋਂ ਕੀ ਹੈ? ਪੜ੍ਹਨ ਦਾ ਮੁੱਖ ਉਦੇਸ਼ 'ਅਰਥ ਬਣਾਉਣਾ' ਹੈ.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('pa.sentence_detector').predict("ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਦੇ ਪੈਰਾਗ੍ਰਾਫਾਂ ਦੇ ਇੱਕ ਮਹਾਨ ਸਰੋਤ ਦੀ ਭਾਲ ਕਰ ਰਹੇ ਹੋ? ਤੁਸੀਂ ਸਹੀ ਜਗ੍ਹਾ ਤੇ ਆਏ ਹੋ. ਇੱਕ ਤਾਜ਼ਾ ਅਧਿਐਨ ਅਨੁਸਾਰ ਅੱਜ ਦੇ ਨੌਜਵਾਨਾਂ ਵਿੱਚ ਪੜ੍ਹਨ ਦੀ ਆਦਤ ਤੇਜ਼ੀ ਨਾਲ ਘਟ ਰਹੀ ਹੈ। ਉਹ ਕੁਝ ਸਕਿੰਟਾਂ ਤੋਂ ਵੱਧ ਸਮੇਂ ਲਈ ਦਿੱਤੇ ਗਏ ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਵਾਲੇ ਪੈਰੇ 'ਤੇ ਧਿਆਨ ਨਹੀਂ ਦੇ ਸਕਦੇ! ਨਾਲ ਹੀ, ਪੜ੍ਹਨਾ ਸਾਰੀਆਂ ਪ੍ਰਤੀਯੋਗੀ ਪ੍ਰੀਖਿਆਵਾਂ ਦਾ ਇੱਕ ਅਨਿੱਖੜਵਾਂ ਅੰਗ ਸੀ ਅਤੇ ਹੈ. ਇਸ ਲਈ, ਤੁਸੀਂ ਆਪਣੇ ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਨੂੰ ਕਿਵੇਂ ਸੁਧਾਰਦੇ ਹੋ? ਇਸ ਪ੍ਰਸ਼ਨ ਦਾ ਉੱਤਰ ਅਸਲ ਵਿੱਚ ਇੱਕ ਹੋਰ ਪ੍ਰਸ਼ਨ ਹੈ: ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਦੀ ਵਰਤੋਂ ਕੀ ਹੈ? ਪੜ੍ਹਨ ਦਾ ਮੁੱਖ ਉਦੇਸ਼ 'ਅਰਥ ਬਣਾਉਣਾ' ਹੈ.", output_level ='sentence') ```

## Results ```bash +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |result | +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਦੇ ਪੈਰਾਗ੍ਰਾਫਾਂ ਦੇ ਇੱਕ ਮਹਾਨ ਸਰੋਤ ਦੀ ਭਾਲ ਕਰ ਰਹੇ ਹੋ?] | |[ਤੁਸੀਂ ਸਹੀ ਜਗ੍ਹਾ ਤੇ ਆਏ ਹੋ.] | |[ਇੱਕ ਤਾਜ਼ਾ ਅਧਿਐਨ ਅਨੁਸਾਰ ਅੱਜ ਦੇ ਨੌਜਵਾਨਾਂ ਵਿੱਚ ਪੜ੍ਹਨ ਦੀ ਆਦਤ ਤੇਜ਼ੀ ਨਾਲ ਘਟ ਰਹੀ ਹੈ। ਉਹ ਕੁਝ ਸਕਿੰਟਾਂ ਤੋਂ ਵੱਧ ਸਮੇਂ ਲਈ ਦਿੱਤੇ ਗਏ ਅੰਗਰੇਜ਼ੀ ਪੜ੍ਹਨ ਵਾਲੇ ਪੈਰੇ 'ਤੇ ਧਿਆਨ ਨਹੀਂ ਦੇ ਸਕਦੇ!]| |[ਨਾਲ ਹੀ, ਪੜ੍ਹਨਾ ਸਾਰੀਆਂ ਪ੍ਰਤੀਯੋਗੀ ਪ੍ਰੀਖਿਆਵਾਂ ਦਾ ਇੱਕ ਅਨਿੱਖੜਵਾਂ ਅੰਗ ਸੀ ਅਤੇ ਹੈ.] | |[ਇਸ ਲਈ, ਤੁਸੀਂ ਆਪਣੇ ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਨੂੰ ਕਿਵੇਂ ਸੁਧਾਰਦੇ ਹੋ?] | |[ਇਸ ਪ੍ਰਸ਼ਨ ਦਾ ਉੱਤਰ ਅਸਲ ਵਿੱਚ ਇੱਕ ਹੋਰ ਪ੍ਰਸ਼ਨ ਹੈ:] | |[ਪੜ੍ਹਨ ਦੇ ਹੁਨਰ ਦੀ ਵਰਤੋਂ ਕੀ ਹੈ?] | |[ਪੜ੍ਹਨ ਦਾ ਮੁੱਖ ਉਦੇਸ਼ 'ਅਰਥ ਬਣਾਉਣਾ' ਹੈ.] | +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|pa| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English asr_wav2vec2_large_robust_libri_960h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_large_robust_libri_960h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_libri_960h` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_robust_libri_960h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_libri_960h_en_4.2.0_3.0_1664039514827.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_libri_960h_en_4.2.0_3.0_1664039514827.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_robust_libri_960h', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_robust_libri_960h", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_robust_libri_960h| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.6 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_wav2vec2_xls_r_300m_german_english TFWav2Vec2ForCTC from aware-ai author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_german_english date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_german_english` is a English model originally trained by aware-ai. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_german_english_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_german_english_en_4.2.0_3.0_1664111961756.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_german_english_en_4.2.0_3.0_1664111961756.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_german_english', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_german_english", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_german_english| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Clean patterns pipeline for English author: John Snow Labs name: clean_pattern date: 2022-07-06 tags: [open_source, english, clean_pattern, pipeline, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The clean_pattern is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/clean_pattern_en_4.0.0_3.0_1657137560119.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/clean_pattern_en_4.0.0_3.0_1657137560119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('clean_pattern', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("clean_pattern", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.clean.pattern').predict(text) result_df ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_base_bulgarian","bg") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_base_bulgarian","bg") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_base_bulgarian| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|bg| |Size:|473.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/iarfmoose/roberta-base-bulgarian - https://arxiv.org/abs/1907.11692 - https://oscar-corpus.com/ - https://wortschatz.uni-leipzig.de/en/download/bulgarian - https://wortschatz.uni-leipzig.de/en/download/bulgarian --- layout: model title: Pipeline to Detect Clinical Conditions (ner_eu_clinical_case - eu) author: John Snow Labs name: ner_eu_clinical_condition_pipeline date: 2023-03-07 tags: [eu, clinical, licensed, ner, clinical_condition] task: Named Entity Recognition language: eu edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_eu_clinical_condition](https://nlp.johnsnowlabs.com/2023/02/06/ner_eu_clinical_condition_eu.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_pipeline_eu_4.3.0_3.2_1678213509285.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_condition_pipeline_eu_4.3.0_3.2_1678213509285.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_eu_clinical_condition_pipeline", "eu", "clinical/models") text = " Gertaera honetatik bi hilabetetara, umea Larrialdietako Zerbitzura dator 4 egunetan zehar buruko mina eta bekokiko hantura azaltzeagatik, sukarrik izan gabe. Miaketan, haztapen mingarria duen bekokiko hantura bigunaz gain, ez da beste zeinurik azaltzen. Polakiuria eta tenesmo arina ere izan zuen egun horretan hematuriarekin batera. Geroztik sintomarik gabe dago. " result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_eu_clinical_condition_pipeline", "eu", "clinical/models") val text = " Gertaera honetatik bi hilabetetara, umea Larrialdietako Zerbitzura dator 4 egunetan zehar buruko mina eta bekokiko hantura azaltzeagatik, sukarrik izan gabe. Miaketan, haztapen mingarria duen bekokiko hantura bigunaz gain, ez da beste zeinurik azaltzen. Polakiuria eta tenesmo arina ere izan zuen egun horretan hematuriarekin batera. Geroztik sintomarik gabe dago. " val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:-----------|--------:|------:|:-------------------|-------------:| | 0 | mina | 98 | 101 | clinical_condition | 0.8754 | | 1 | hantura | 116 | 122 | clinical_condition | 0.8877 | | 2 | sukarrik | 139 | 146 | clinical_condition | 0.9119 | | 3 | mingarria | 178 | 186 | clinical_condition | 0.7381 | | 4 | hantura | 203 | 209 | clinical_condition | 0.8805 | | 5 | Polakiuria | 256 | 265 | clinical_condition | 0.6683 | | 6 | sintomarik | 345 | 354 | clinical_condition | 0.9632 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_condition_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|eu| |Size:|1.1 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English image_classifier_vit_anomaly ViTForImageClassification from hafidber author: John Snow Labs name: image_classifier_vit_anomaly date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_anomaly` is a English model originally trained by hafidber. ## Predicted Entities `abnormal`, `normal` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_anomaly_en_4.1.0_3.0_1660169901656.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_anomaly_en_4.1.0_3.0_1660169901656.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_anomaly", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_anomaly", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_anomaly| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Supplemental Indenture Document Binary Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_supplemental_indenture_agreement_bert date: 2022-12-18 tags: [en, legal, classification, licensed, document, bert, supplemental, indenture, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_supplemental_indenture_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `supplemental-indenture` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `supplemental-indenture`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_supplemental_indenture_agreement_bert_en_1.0.0_3.0_1671393857190.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_supplemental_indenture_agreement_bert_en_1.0.0_3.0_1671393857190.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_supplemental_indenture_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[supplemental-indenture]| |[other]| |[other]| |[supplemental-indenture]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_supplemental_indenture_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.95 0.96 204 supplemental-indenture 0.91 0.95 0.93 111 accuracy - - 0.95 315 macro-avg 0.94 0.95 0.95 315 weighted-avg 0.95 0.95 0.95 315 ``` --- layout: model title: English image_classifier_vit_deit_flyswot ViTForImageClassification from davanstrien author: John Snow Labs name: image_classifier_vit_deit_flyswot date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_deit_flyswot` is a English model originally trained by davanstrien. ## Predicted Entities `EDGE + SPINE`, `OTHER`, `PAGE + FOLIO`, `FLYSHEET`, `CONTAINER`, `CONTROL SHOT`, `COVER`, `SCROLL` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_flyswot_en_4.1.0_3.0_1660166402706.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_flyswot_en_4.1.0_3.0_1660166402706.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_deit_flyswot", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_deit_flyswot", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_deit_flyswot| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.8 MB| --- layout: model title: English image_classifier_vit_llama_alpaca_snake ViTForImageClassification from osanseviero author: John Snow Labs name: image_classifier_vit_llama_alpaca_snake date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_llama_alpaca_snake` is a English model originally trained by osanseviero. ## Predicted Entities `alpaca`, `llamas`, `snake` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_llama_alpaca_snake_en_4.1.0_3.0_1660170191761.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_llama_alpaca_snake_en_4.1.0_3.0_1660170191761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_llama_alpaca_snake", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_llama_alpaca_snake", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_llama_alpaca_snake| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab66 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab66 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab66` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab66_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab66_en_4.2.0_3.0_1664024830506.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab66_en_4.2.0_3.0_1664024830506.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab66", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab66", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab66| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Translate English to Tetela Pipeline author: John Snow Labs name: translate_en_tll date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tll, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tll` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tll_xx_2.7.0_2.4_1609699282699.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tll_xx_2.7.0_2.4_1609699282699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tll", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tll", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tll').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tll| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Base Uncased model (from ksabeh) author: John Snow Labs name: bert_qa_base_uncased_attribute_correction_mlm_titles date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-attribute-correction-mlm-titles` is a English model originally trained by `ksabeh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_attribute_correction_mlm_titles_en_4.0.0_3.0_1657183812655.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_attribute_correction_mlm_titles_en_4.0.0_3.0_1657183812655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_attribute_correction_mlm_titles","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_attribute_correction_mlm_titles","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_attribute_correction_mlm_titles| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ksabeh/bert-base-uncased-attribute-correction-mlm-titles --- layout: model title: Lewotobi RobertaForQuestionAnswering (from 21iridescent) author: John Snow Labs name: roberta_qa_distilroberta_base_finetuned_squad2_lwt date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: lwt edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-base-finetuned-squad2-lwt` is a Lewotobi model originally trained by `21iridescent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_base_finetuned_squad2_lwt_lwt_4.0.0_3.0_1655728304909.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_base_finetuned_squad2_lwt_lwt_4.0.0_3.0_1655728304909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilroberta_base_finetuned_squad2_lwt","lwt") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_distilroberta_base_finetuned_squad2_lwt","lwt") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("lwt.answer_question.squadv2.roberta.distilled_base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilroberta_base_finetuned_squad2_lwt| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|lwt| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/21iridescent/distilroberta-base-finetuned-squad2-lwt --- layout: model title: French BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_fr_cased date: 2022-12-02 tags: [fr, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: fr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-fr-cased` is a French model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_fr_cased_fr_4.2.4_3.0_1670017613286.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_fr_cased_fr_4.2.4_3.0_1670017613286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_fr_cased","fr") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_fr_cased","fr") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_fr_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fr| |Size:|393.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-fr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Pipeline to Detect Chemicals in Medical text (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_chemicals_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, berfortokenclassification, chemicals, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemicals](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemicals_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_3.0_1647889424974.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_3.0_1647889424974.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemicals_pipeline").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |p - choloroaniline |CHEM | |chlorhexidine - digluconate|CHEM | |kanamycin |CHEM | |colistin |CHEM | |povidone - iodine |CHEM | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverter --- layout: model title: English image_classifier_vit_vision_transformer_v3 ViTForImageClassification from mrgiraffe author: John Snow Labs name: image_classifier_vit_vision_transformer_v3 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vision_transformer_v3` is a English model originally trained by mrgiraffe. ## Predicted Entities `chart`, `imagechart`, `notchart`, `pdfpagechart` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformer_v3_en_4.1.0_3.0_1660168553159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformer_v3_en_4.1.0_3.0_1660168553159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vision_transformer_v3", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vision_transformer_v3", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vision_transformer_v3| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Hindi XlmRoBertaForQuestionAnswering (from bhavikardeshna) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_hindi date: 2022-06-23 tags: [hi, open_source, question_answering, xlmroberta] task: Question Answering language: hi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-hindi` is a Hindi model originally trained by `bhavikardeshna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_hindi_hi_4.0.0_3.0_1655990042129.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_hindi_hi_4.0.0_3.0_1655990042129.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_hindi","hi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_hindi","hi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("hi.answer_question.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_hindi| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|hi| |Size:|885.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bhavikardeshna/xlm-roberta-base-hindi --- layout: model title: Finnish asr_wav2vec2_xlsr_train_aug_bigLM_1B TFWav2Vec2ForCTC from RASMUS author: John Snow Labs name: asr_wav2vec2_xlsr_train_aug_bigLM_1B date: 2022-09-25 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_train_aug_bigLM_1B` is a Finnish model originally trained by RASMUS. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_train_aug_bigLM_1B_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_train_aug_bigLM_1B_fi_4.2.0_3.0_1664097486875.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_train_aug_bigLM_1B_fi_4.2.0_3.0_1664097486875.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_train_aug_bigLM_1B", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_train_aug_bigLM_1B", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_train_aug_bigLM_1B| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|3.6 GB| --- layout: model title: German asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779 date: 2022-09-26 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779_de_4.2.0_3.0_1664191745283.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779_de_4.2.0_3.0_1664191745283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s779| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Legal Enforceability Clause Binary Classifier author: John Snow Labs name: legclf_enforceability_clause date: 2022-09-28 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `enforceability` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `enforceability` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_enforceability_clause_en_1.0.0_3.0_1664363141173.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_enforceability_clause_en_1.0.0_3.0_1664363141173.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_enforceability_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[enforceability]| |[other]| |[other]| |[enforceability]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_enforceability_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support enforceability 0.87 0.89 0.88 38 other 0.95 0.94 0.94 78 accuracy - - 0.92 116 macro-avg 0.91 0.92 0.91 116 weighted-avg 0.92 0.92 0.92 116 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Modified_BlueBERT_512 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Modified-BlueBERT-512` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_BlueBERT_512_en_4.0.0_3.0_1657108398297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_BlueBERT_512_en_4.0.0_3.0_1657108398297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Modified_BlueBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Modified-BlueBERT-512 --- layout: model title: English BertForQuestionAnswering model (from rahulkuruvilla) author: John Snow Labs name: bert_qa_COVID_BERTa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `COVID-BERTa` is a English model orginally trained by `rahulkuruvilla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_COVID_BERTa_en_4.0.0_3.0_1654176515744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_COVID_BERTa_en_4.0.0_3.0_1654176515744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_COVID_BERTa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_COVID_BERTa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.covid_bert.a.by_rahulkuruvilla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_COVID_BERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rahulkuruvilla/COVID-BERTa --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman_de_4.2.0_3.0_1664096062320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman_de_4.2.0_3.0_1664096062320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_german_by_jonatasgrosman| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate English to Dravidian languages Pipeline author: John Snow Labs name: translate_en_dra date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, dra, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `dra` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_dra_xx_2.7.0_2.4_1609698809869.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_dra_xx_2.7.0_2.4_1609698809869.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_dra", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_dra", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.dra').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_dra| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Hindi Bert Embeddings (from Geotrend) author: John Snow Labs name: bert_embeddings_bert_base_hi_cased date: 2022-04-11 tags: [bert, embeddings, hi, open_source] task: Embeddings language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-hi-cased` is a Hindi model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_hi_cased_hi_3.4.2_3.0_1649673139297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_hi_cased_hi_3.4.2_3.0_1649673139297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_hi_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|339.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-hi-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Mapping HCPCS Codes with Corresponding National Drug Codes (NDC) and Drug Brand Names author: John Snow Labs name: hcpcs_ndc_mapper date: 2023-04-13 tags: [en, licensed, chunk_mappig, hcpcs, ndc, brand_name] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps HCPCS codes with their corresponding National Drug Codes (NDC) and their drug brand names. ## Predicted Entities `ndc_code`, `brand_name` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/hcpcs_ndc_mapper_en_4.4.0_3.0_1681405950608.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/hcpcs_ndc_mapper_en_4.4.0_3.0_1681405950608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("hcpcs_chunk") chunkerMapper = DocMapperModel.pretrained("hcpcs_ndc_mapper", "en", "clinical/models")\ .setInputCols(["hcpcs_chunk"])\ .setOutputCol("mappings")\ .setRels(["ndc_code", "brand_name"]) pipeline = Pipeline().setStages([document_assembler, chunkerMapper]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) res = lp.fullAnnotate(["Q5106", "J9211", "J7508"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text")\ .setOutputCol("hcpcs_chunk") val chunkerMapper = DocMapperModel .pretrained("hcpcs_ndc_mapper", "en", "clinical/models") .setInputCols(Array("hcpcs_chunk")) .setOutputCol("mappings") .setRels(Array("ndc_code", "brand_name")) val mapper_pipeline = new Pipeline().setStages(Array( document_assembler, chunkerMapper)) val data = Seq(Array(["Q5106", "J9211", "J7508"])).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-----------+-------------------------------------+----------+ |hcpcs_chunk|mappings |relation | +-----------+-------------------------------------+----------+ |Q5106 |59353-0003-10 |ndc_code | |Q5106 |RETACRIT (PF) 3000 U/1 ML |brand_name| |J9211 |59762-2596-01 |ndc_code | |J9211 |IDARUBICIN HYDROCHLORIDE (PF) 1 MG/ML|brand_name| |J7508 |00469-0687-73 |ndc_code | |J7508 |ASTAGRAF XL 5 MG |brand_name| +-----------+-------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|hcpcs_ndc_mapper| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|20.7 KB| --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_oliverguhr TFWav2Vec2ForCTC from oliverguhr author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_oliverguhr` is a German model originally trained by oliverguhr. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_de_4.2.0_3.0_1664104534440.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_de_4.2.0_3.0_1664104534440.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_german_by_oliverguhr| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Information Technology And Data Processing Document Classifier (EURLEX) author: John Snow Labs name: legclf_information_technology_and_data_processing_bert date: 2023-03-06 tags: [en, legal, classification, clauses, information_technology_and_data_processing, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_information_technology_and_data_processing_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Information_Technology_and_Data_Processing or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Information_Technology_and_Data_Processing`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_information_technology_and_data_processing_bert_en_1.0.0_3.0_1678111859916.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_information_technology_and_data_processing_bert_en_1.0.0_3.0_1678111859916.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_information_technology_and_data_processing_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Information_Technology_and_Data_Processing]| |[Other]| |[Other]| |[Information_Technology_and_Data_Processing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_information_technology_and_data_processing_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Information_Technology_and_Data_Processing 0.85 0.80 0.82 153 Other 0.79 0.85 0.82 141 accuracy - - 0.82 294 macro-avg 0.82 0.82 0.82 294 weighted-avg 0.83 0.82 0.82 294 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_7 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-7` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_7_en_4.3.0_3.0_1672767624756.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_7_en_4.3.0_3.0_1672767624756.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_7","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_7","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_7| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-7 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from clisi2000) author: John Snow Labs name: xlmroberta_ner_clisi2000_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `clisi2000`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_clisi2000_base_finetuned_panx_de_4.1.0_3.0_1660431788167.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_clisi2000_base_finetuned_panx_de_4.1.0_3.0_1660431788167.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_clisi2000_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_clisi2000_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_clisi2000_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/clisi2000/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English DistilBertForQuestionAnswering model (from minhdang241) author: John Snow Labs name: distilbert_qa_robustqa_baseline_01 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robustqa-baseline-01` is a English model originally trained by `minhdang241`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_robustqa_baseline_01_en_4.0.0_3.0_1654728524154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_robustqa_baseline_01_en_4.0.0_3.0_1654728524154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_robustqa_baseline_01","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_robustqa_baseline_01","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base.by_minhdang241").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_robustqa_baseline_01| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/minhdang241/robustqa-baseline-01 --- layout: model title: French CamemBert Embeddings (from yancong) author: John Snow Labs name: camembert_embeddings_yancong_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `yancong`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_yancong_generic_model_fr_3.4.4_3.0_1653990825081.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_yancong_generic_model_fr_3.4.4_3.0_1653990825081.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_yancong_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/yancong/dummy-model --- layout: model title: Chinese BertForMaskedLM Cased model (from qinluo) author: John Snow Labs name: bert_embeddings_wo_chinese_plus date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `wobert-chinese-plus` is a Chinese model originally trained by `qinluo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_wo_chinese_plus_zh_4.2.4_3.0_1670023089360.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_wo_chinese_plus_zh_4.2.4_3.0_1670023089360.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_wo_chinese_plus","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_wo_chinese_plus","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_wo_chinese_plus| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|467.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/qinluo/wobert-chinese-plus - https://github.com/ZhuiyiTechnology/WoBERT - https://github.com/JunnYu/WoBERT_pytorch --- layout: model title: English DistilBertForQuestionAnswering model (from abhilash1910) author: John Snow Labs name: distilbert_qa_squadv1 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-squadv1` is a English model originally trained by `abhilash1910`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squadv1_en_4.0.0_3.0_1654727758712.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squadv1_en_4.0.0_3.0_1654727758712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.by_abhilash1910").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/abhilash1910/distilbert-squadv1 --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes (Drug & Substance) author: John Snow Labs name: sbiobertresolve_umls_drug_substance date: 2021-12-06 tags: [entity_resolution, en, clinical, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.3 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities to UMLS CUI codes. It is trained on `2021AB` UMLS dataset. The complete dataset has 127 different categories, and this model is trained on the `Clinical Drug`, `Pharmacologic Substance`, `Antibiotic`, `Hazardous or Poisonous Substance` categories using `sbiobert_base_cased_mli` embeddings. ## Predicted Entities `Predicts UMLS codes for Drugs & Substances medical concepts` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_drug_substance_en_3.3.3_3.0_1638802613409.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_drug_substance_en_3.3.3_3.0_1638802613409.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentenceDetector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") stopwords = StopWordsCleaner.pretrained()\ .setInputCols("token")\ .setOutputCol("cleanTokens")\ .setCaseSensitive(False) word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "cleanTokens"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "cleanTokens", "ner"]) \ .setOutputCol("ner_chunk") chunk2doc = Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli",'en','clinical/models')\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings").setCaseSensitive(False) resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_umls_drug_substance","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, resolver]) data = spark.createDataFrame([[""]]).toDF("text") model = LightPipeline(pipeline.fit(data)) results = model.fullAnnotate(['Dilaudid', 'Hydromorphone', 'Exalgo', 'Palladone', 'Hydrogen peroxide 30 mg', 'Neosporin Cream', 'Magnesium hydroxide 100mg/1ml', 'Metformin 1000 mg']) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val stopwords = StopWordsCleaner.pretrained() .setInputCols("token") .setOutputCol("cleanTokens") .setCaseSensitive(False) val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "cleanTokens")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "cleanTokens", "ner")) .setOutputCol("ner_chunk") val chunk2doc = new Chunk2Doc() .setInputCols("ner_chunk") .setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols("ner_chunk_doc") .setOutputCol("sbert_embeddings") val resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_umls_drug_substance", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val p_model = new PipelineModel().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, resolver)) val data = Seq("""'Dilaudid', 'Hydromorphone', 'Exalgo', 'Palladone', 'Hydrogen peroxide 30 mg', 'Neosporin Cream', 'Magnesium hydroxide 100mg/1ml', 'Metformin 1000 mg'""").toDS().toDF("text") val res = p_model.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.umls_drug_substance").predict("""Magnesium hydroxide 100mg/1ml""") ```

## Results ```bash | | chunk | code | code_description | all_k_code_desc | all_k_codes | |---:|:------------------------------|:---------|:---------------------------|:-------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | Dilaudid | C0728755 | dilaudid | ['C0728755', 'C0719907', 'C1448344', 'C0305924', 'C1569295'] | ['dilaudid', 'Dilaudid HP', 'Disthelm', 'Dilaudid Injection', 'Distaph'] | | 1 | Hydromorphone | C0012306 | HYDROMORPHONE | ['C0012306', 'C0700533', 'C1646274', 'C1170495', 'C0498841'] | ['HYDROMORPHONE', 'Hydromorphone HCl', 'Phl-HYDROmorphone', 'PMS HYDROmorphone', 'Hydromorphone injection'] | | 2 | Exalgo | C2746500 | Exalgo | ['C2746500', 'C0604734', 'C1707065', 'C0070591', 'C3660437'] | ['Exalgo', 'exaltolide', 'Exelgyn', 'Extacol', 'exserohilone'] | | 3 | Palladone | C0730726 | palladone | ['C0730726', 'C0594402', 'C1655349', 'C0069952', 'C2742475'] | ['palladone', 'Palladone-SR', 'Palladone IR', 'palladiazo', 'palladia'] | | 4 | Hydrogen peroxide 30 mg | C1126248 | hydrogen peroxide 30 MG/ML | ['C1126248', 'C0304655', 'C1605252', 'C0304656', 'C1154260'] | ['hydrogen peroxide 30 MG/ML', 'Hydrogen peroxide solution 30%', 'hydrogen peroxide 30 MG/ML [Proxacol]', 'Hydrogen peroxide 30 mg/mL cutaneous solution', 'benzoyl peroxide 30 MG/ML'] | | 5 | Neosporin Cream | C0132149 | Neosporin Cream | ['C0132149', 'C0306959', 'C4722788', 'C0704071', 'C0698988'] | ['Neosporin Cream', 'Neosporin Ointment', 'Neomycin Sulfate Cream', 'Neosporin Topical Ointment', 'Naseptin cream'] | | 6 | Magnesium hydroxide 100mg/1ml | C1134402 | magnesium hydroxide 100 MG | ['C1134402', 'C1126785', 'C4317023', 'C4051486', 'C4047137'] | ['magnesium hydroxide 100 MG', 'magnesium hydroxide 100 MG/ML', 'Magnesium sulphate 100mg/mL injection', 'magnesium sulfate 100 MG', 'magnesium sulfate 100 MG/ML'] | | 7 | Metformin 1000 mg | C0987664 | metformin 1000 MG | ['C0987664', 'C2719784', 'C0978482', 'C2719786', 'C4282269'] | ['metformin 1000 MG', 'metFORMIN hydrochloride 1000 MG', 'METFORMIN HCL 1000MG TAB', 'metFORMIN hydrochloride 1000 MG [Fortamet]', 'METFORMIN HCL 1000MG SA TAB'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_drug_substance| |Compatibility:|Healthcare NLP 3.3.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[output]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on `2021AB` UMLS dataset’s `Clinical Drug`, `Pharmacologic Substance`, `Antibiotic`, `Hazardous or Poisonous Substance` categories. https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: Italian Embeddings (Base, Wines description) author: John Snow Labs name: bert_embeddings_wineberto_italian_cased date: 2022-04-11 tags: [bert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `wineberto-italian-cased` is a Italian model orginally trained by `vinhood`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_wineberto_italian_cased_it_3.4.2_3.0_1649676965822.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_wineberto_italian_cased_it_3.4.2_3.0_1649676965822.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_wineberto_italian_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|415.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/vinhood/wineberto-italian-cased - https://twitter.com/denocris - https://www.linkedin.com/in/cristiano-de-nobili/ - https://www.vinhood.com/en/ --- layout: model title: Legal Purchase price Clause Binary Classifier author: John Snow Labs name: legclf_purchase_price_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `purchase-price` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `purchase-price` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_purchase_price_clause_en_1.0.0_3.2_1660123871318.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_purchase_price_clause_en_1.0.0_3.2_1660123871318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_purchase_price_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[purchase-price]| |[other]| |[other]| |[purchase-price]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_purchase_price_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.99 0.99 81 purchase-price 0.98 0.98 0.98 47 accuracy - - 0.98 128 macro-avg 0.98 0.98 0.98 128 weighted-avg 0.98 0.98 0.98 128 ``` --- layout: model title: Pipeline to Detect PHI for deidentification purposes author: John Snow Labs name: ner_deid_subentity_augmented_i2b2_pipeline date: 2023-03-13 tags: [deid, ner, phi, deidentification, licensed, i2b2, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_subentity_augmented_i2b2](https://nlp.johnsnowlabs.com/2021/11/29/ner_deid_subentity_augmented_i2b2_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_i2b2_pipeline_en_4.3.0_3.2_1678735152629.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_i2b2_pipeline_en_4.3.0_3.2_1678735152629.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_subentity_augmented_i2b2_pipeline", "en", "clinical/models") text = '''Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_subentity_augmented_i2b2_pipeline", "en", "clinical/models") val text = "Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.subentity_ner_augmented_i2b2.pipeline").predict("""Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:--------------|-------------:| | 0 | 2093-01-13 | 14 | 23 | DATE | 0.9997 | | 1 | David Hale | 27 | 36 | DOCTOR | 0.9507 | | 2 | Hendrickson Ora | 55 | 69 | PATIENT | 0.9981 | | 3 | 7194334 | 78 | 84 | MEDICALRECORD | 0.9996 | | 4 | 01/13/93 | 93 | 100 | DATE | 0.9992 | | 5 | Oliveira | 110 | 117 | DOCTOR | 0.8822 | | 6 | 25 | 121 | 122 | AGE | 0.5648 | | 7 | 2079-11-09 | 150 | 159 | DATE | 0.9995 | | 8 | Cocke County Baptist Hospital | 163 | 191 | HOSPITAL | 0.863775 | | 9 | 0295 Keats Street | 195 | 211 | STREET | 0.754533 | | 10 | 302-786-5227 | 221 | 232 | PHONE | 0.9697 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_augmented_i2b2_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Chinese Bert Embeddings (Base) author: John Snow Labs name: bert_embeddings_bert_base_chinese_jinyong date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-chinese-jinyong` is a Chinese model orginally trained by `yechen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_chinese_jinyong_zh_3.4.2_3.0_1649670833638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_chinese_jinyong_zh_3.4.2_3.0_1649670833638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_chinese_jinyong| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|384.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/yechen/bert-base-chinese-jinyong --- layout: model title: Detect Anatomical Regions author: John Snow Labs name: ner_anatomy_en date: 2020-04-22 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [ner, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for anatomy terms. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `Anatomical_system`, `Cell`, `Cellular_component`, `Developing_anatomical_structure`, `Immaterial_anatomical_entity`, `Multi-tissue_structure`, `Organ`, `Organism_subdivision`, `Organism_substance`, `Pathological_formation`, `Tissue` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ANATOMY/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_en_2.4.2_2.4_1587513307751.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_en_2.4.2_2.4_1587513307751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPython.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_anatomy", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) results = model.transform(spark.createDataFrame([['This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along the lateral portion of her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.']], ["text"])) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_anatomy", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val result = pipeline.fit(Seq.empty["This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist. Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along the lateral portion of her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short."].toDS.toDF("text")).transform(data) ```

## Results ```bash Masked with entity labels ------------------------------ Dados do . Nome: . Apelido: . NIF: . NISS: . Endereço: . CÓDIGO POSTAL: . Dados de cuidados. Data de nascimento: . País: . Idade: anos Sexo: . Data de admissão: . Doutor: Cuéllar NºCol: . Relatório clínico do : de anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: - , 22 E-mail: . Masked with chars ------------------------------ Dados do [******]. Nome: [***]. Apelido: [*******]. NIF: [****]. NISS: [*********]. Endereço: [*********************]. CÓDIGO POSTAL: [***]. Dados de cuidados. Data de nascimento: [********]. País: [******]. Idade: ** anos Sexo: *. Data de admissão: [********]. Doutor: [*************] Cuéllar NºCol: ** ** [***]. Relatório clínico do [******]: [******] de ** anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér[**] de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: [***********] - [*****************], 22 [******] E-mail: [****************]. Masked with fixed length chars ------------------------------ Dados do ****. Nome: ****. Apelido: ****. NIF: ****. NISS: ****. Endereço: ****. CÓDIGO POSTAL: ****. Dados de cuidados. Data de nascimento: ****. País: ****. Idade: **** anos Sexo: ****. Data de admissão: ****. Doutor: **** Cuéllar NºCol: **** **** ****. Relatório clínico do ****: **** de **** anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér**** de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: **** - ****, 22 **** E-mail: ****. Obfuscated ------------------------------ Dados do H.. Nome: Marcos Alves. Apelido: Tiago Santos. NIF: 566-445. NISS: 134544332. Endereço: Rua de Santa María, 100. CÓDIGO POSTAL: 4099. Dados de cuidados. Data de nascimento: 31/03/1946. País: Espanha. Idade: 46 anos Sexo: Mulher. Data de admissão: 06/01/2017. Doutor: Carlos Melo Cuéllar NºCol: 134544332 134544332 124 445 311. Relatório clínico do H.: M. de 46 anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicérHomen de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: Carlos Melo - Avenida Dos Aliados, 56, 22 Espanha E-mail: maria.prado@jacob.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|1.3 GB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetectorDLModel - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - nlp.NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - nlp.TextMatcherModel - ContextualParserModel - ContextualParserModel - nlp.RegexMatcherModel - nlp.RegexMatcherModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77 TFWav2Vec2ForCTC from emeson77 author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77` is a English model originally trained by emeson77. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_en_4.2.0_3.0_1664037180334.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_en_4.2.0_3.0_1664037180334.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Legal Food Technology Document Classifier (EURLEX) author: John Snow Labs name: legclf_food_technology_bert date: 2023-03-06 tags: [en, legal, classification, clauses, food_technology, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_food_technology_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Food_Technology or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Food_Technology`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_food_technology_bert_en_1.0.0_3.0_1678111847548.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_food_technology_bert_en_1.0.0_3.0_1678111847548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_food_technology_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Food_Technology]| |[Other]| |[Other]| |[Food_Technology]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_food_technology_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Food_Technology 0.87 0.86 0.86 221 Other 0.83 0.84 0.84 181 accuracy - - 0.85 402 macro-avg 0.85 0.85 0.85 402 weighted-avg 0.85 0.85 0.85 402 ``` --- layout: model title: Vaccine Sentiment Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_vaccine_sentiment date: 2022-07-28 tags: [public_health, vaccine_sentiment, en, licensed, sequence_classification] task: Sentiment Analysis language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true recommended: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT](https://nlp.johnsnowlabs.com/2022/07/18/biobert_pubmed_base_cased_v1.2_en_3_0.html) based sentimental analysis model that can extract information from COVID-19 Vaccine-related tweets. The model predicts whether a tweet contains positive, negative, or neutral sentiments about COVID-19 Vaccines. ## Predicted Entities `neutral`, `positive`, `negative` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_VACCINE_STATUS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vaccine_sentiment_en_4.0.0_3.0_1658995472179.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vaccine_sentiment_en_4.0.0_3.0_1658995472179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------------------------------------------------------------------+----------+ |text |class | +-----------------------------------------------------------------------------------------------------+----------+ |A little bright light for an otherwise dark week. Thanks researchers, and frontline workers. Onwards.|[positive]| |People with a history of severe allergic reaction to any component of the vaccine should not take. |[negative]| |43 million doses of vaccines administrated worldwide...Production capacity of CHINA to reach 4 b |[neutral] | +-----------------------------------------------------------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vaccine_sentiment| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References Curated from several academic and in-house datasets. ## Benchmarking ```bash label precision recall f1-score support neutral 0.82 0.78 0.80 1007 positive 0.88 0.90 0.89 1002 negative 0.83 0.86 0.84 881 accuracy - - 0.85 2890 macro-avg 0.85 0.85 0.85 2890 weighted-avg 0.85 0.85 0.85 2890 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_only_classfn_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191247552.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191247552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_only_classfn_epochs_1_shard_1_squad2.0 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from Leizhang) author: John Snow Labs name: xlmroberta_ner_leizhang_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `Leizhang`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_leizhang_base_finetuned_panx_de_4.1.0_3.0_1660429672416.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_leizhang_base_finetuned_panx_de_4.1.0_3.0_1660429672416.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_leizhang_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_leizhang_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_leizhang_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Leizhang/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English asr_wav2vec2_xls_r_300m_Turkish_Tr_med TFWav2Vec2ForCTC from emre author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_Turkish_Tr_med` is a English model originally trained by emre. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med_en_4.2.0_3.0_1664037839645.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med_en_4.2.0_3.0_1664037839645.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_Turkish_Tr_med| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate English to Salishan languages Pipeline author: John Snow Labs name: translate_en_sal date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, sal, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `sal` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_sal_xx_2.7.0_2.4_1609686644399.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_sal_xx_2.7.0_2.4_1609686644399.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_sal", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_sal", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.sal').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_sal| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Seniority Clause Binary Classifier author: John Snow Labs name: legclf_seniority_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `seniority` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `seniority` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_seniority_clause_en_1.0.0_3.2_1660123984376.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_seniority_clause_en_1.0.0_3.2_1660123984376.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_seniority_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[seniority]| |[other]| |[other]| |[seniority]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_seniority_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.96 0.97 91 seniority 0.90 0.97 0.94 37 accuracy - - 0.96 128 macro-avg 0.94 0.96 0.95 128 weighted-avg 0.96 0.96 0.96 128 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465522 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465522` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465522_en_4.0.0_3.0_1655986870269.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465522_en_4.0.0_3.0_1655986870269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465522","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465522","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465522.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465522| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465522 --- layout: model title: English RobertaForQuestionAnswering (from LucasS) author: John Snow Labs name: roberta_qa_robertaBaseABSA date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robertaBaseABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaBaseABSA_en_4.0.0_3.0_1655738733590.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaBaseABSA_en_4.0.0_3.0_1655738733590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertaBaseABSA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_robertaBaseABSA","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_robertaBaseABSA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|436.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LucasS/robertaBaseABSA --- layout: model title: English BertForQuestionAnswering model (from horsbug98) author: John Snow Labs name: bert_qa_Part_2_mBERT_Model_E2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Part_2_mBERT_Model_E2` is a English model orginally trained by `horsbug98`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Part_2_mBERT_Model_E2_en_4.0.0_3.0_1654178989285.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Part_2_mBERT_Model_E2_en_4.0.0_3.0_1654178989285.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Part_2_mBERT_Model_E2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Part_2_mBERT_Model_E2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.tydiqa.multi_lingual_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Part_2_mBERT_Model_E2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/horsbug98/Part_2_mBERT_Model_E2 --- layout: model title: English RobertaForQuestionAnswering Cased model (from comacrae) author: John Snow Labs name: roberta_qa_eda_and_parav3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-eda-and-parav3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_eda_and_parav3_en_4.3.0_3.0_1674220091398.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_eda_and_parav3_en_4.3.0_3.0_1674220091398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_eda_and_parav3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_eda_and_parav3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_eda_and_parav3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/comacrae/roberta-eda-and-parav3 --- layout: model title: English image_classifier_vit_dog_food__base_patch16_224_in21k ViTForImageClassification from sasha author: John Snow Labs name: image_classifier_vit_dog_food__base_patch16_224_in21k date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_dog_food__base_patch16_224_in21k` is a English model originally trained by sasha. ## Predicted Entities `dog`, `food` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_food__base_patch16_224_in21k_en_4.1.0_3.0_1660171837844.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_food__base_patch16_224_in21k_en_4.1.0_3.0_1660171837844.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_dog_food__base_patch16_224_in21k", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_dog_food__base_patch16_224_in21k", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_dog_food__base_patch16_224_in21k| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Sentence Embeddings - sbert medium (tuned) author: John Snow Labs name: sbert_jsl_medium_rxnorm_uncased date: 2022-01-03 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps sentences & documents to a 512-dimensional dense vector space by using average pooling on top of BERT model. It's also fine-tuned on the RxNorm dataset to help generalization over medication-related datasets. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_rxnorm_uncased_en_3.3.4_2.4_1641241051941.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_rxnorm_uncased_en_3.3.4_2.4_1641241051941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_rxnorm_uncased", "en", "clinical/models")\ .setInputCols("sentence")\ .setOutputCol("sbert_embeddings") ``` ```scala val sentence_embeddings = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_rxnorm_uncased", "en", "clinical/models") .setInputCols("sentence") .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert_uncased.rxnorm").predict("""Put your text here.""") ```

## Results ```bash Gives a 512-dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_medium_rxnorm_uncased| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|153.9 MB| |Case sensitive:|false| --- layout: model title: English asr_wav2vec2_large_a TFWav2Vec2ForCTC from yongjian author: John Snow Labs name: asr_wav2vec2_large_a date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_a` is a English model originally trained by yongjian. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_a_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_a_en_4.2.0_3.0_1664039889685.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_a_en_4.2.0_3.0_1664039889685.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_a", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_a", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_a| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_10_h_512 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-10_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_512_zh_4.2.4_3.0_1670021484393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_512_zh_4.2.4_3.0_1670021484393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_10_h_512| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|161.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-10_H-512 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Detect Assertion Status (assertion_jsl_augmented) author: John Snow Labs name: assertion_jsl_augmented date: 2022-09-15 tags: [licensed, clinical, assertion, en] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The deep neural network architecture for assertion status detection in Spark NLP is based on a BiLSTM framework, and it is a modified version of the architecture proposed by Fancellu et.al. (Fancellu, Lopez, and Webber 2016). Its goal is to classify the assertions made on given medical concepts as being present, absent, or possible in the patient, conditionally present in the patient under certain circumstances, hypothetically present in the patient at some future point, and mentioned in the patient report but associated with someoneelse (Uzuner et al. 2011). This model is also the augmented version of [assertion_jsl](https://nlp.johnsnowlabs.com/2021/07/24/assertion_jsl_en.html) model with in-house annotations and it returns confidence scores of the results. ## Predicted Entities `Present`, `Absent`, `Possible`, `Planned`, `Past`, `Family`, `Hypotetical`, `SomeoneElse` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_jsl_augmented_en_4.1.0_3.0_1663252918565.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_jsl_augmented_en_4.1.0_3.0_1663252918565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------+-----+---+-------------------------+-----------+---------+ |ner_chunk |begin|end|ner_label |sentence_id|assertion| +--------------+-----+---+-------------------------+-----------+---------+ |headache |14 |21 |Symptom |0 |Past | |anxious |57 |63 |Symptom |0 |Possible | |alopecia |89 |96 |Disease_Syndrome_Disorder|1 |Absent | |pain |116 |119|Symptom |2 |Absent | |paralyzed |136 |144|Symptom |3 |Family | |antidepressant|212 |225|Drug_Ingredient |4 |Past | |sleeping pills|242 |255|Drug_Ingredient |5 |Planned | |insomnia |273 |280|Symptom |5 |Present | +--------------+-----+---+-------------------------+-----------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_jsl_augmented| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|6.5 MB| ## Benchmarking ```bash label precision recall f1-score Absent 0.94 0.93 0.94 Family 0.88 0.91 0.89 Hypothetical 0.85 0.82 0.83 Past 0.89 0.89 0.89 Planned 0.78 0.81 0.80 Possible 0.82 0.82 0.82 Present 0.91 0.93 0.92 SomeoneElse 0.88 0.80 0.84 ``` --- layout: model title: Explain Document pipeline for Swedish (explain_document_lg) author: John Snow Labs name: explain_document_lg date: 2021-03-23 tags: [open_source, swedish, explain_document_lg, pipeline, sv] supported: true task: [Named Entity Recognition, Lemmatization] language: sv edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_lg_sv_3.0.0_3.0_1616520973696.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_lg_sv_3.0.0_3.0_1616520973696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_lg', lang = 'sv') annotations = pipeline.fullAnnotate(""Hej från John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_lg", lang = "sv") val result = pipeline.fullAnnotate("Hej från John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hej från John Snow Labs! ""] result_df = nlu.load('sv.explain.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:------------------------------|:-----------------------------|:-----------------------------------------|:-----------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hej från John Snow Labs! '] | ['Hej från John Snow Labs!'] | ['Hej', 'från', 'John', 'Snow', 'Labs!'] | ['Hej', 'från', 'John', 'Snow', 'Labs!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.0306969992816448,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|sv| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Lolaibrin) author: John Snow Labs name: distilbert_qa_lolaibrin_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Lolaibrin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_lolaibrin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768653496.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_lolaibrin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768653496.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lolaibrin_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lolaibrin_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_lolaibrin_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Lolaibrin/distilbert-base-uncased-finetuned-squad --- layout: model title: Longformer Base NER Pipeline author: ahmedlone127 name: longformer_base_token_classifier_conll03_pipeline date: 2022-06-14 tags: [ner, longformer, pipeline, conll, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [longformer_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/10/09/longformer_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/longformer_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655213912525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/longformer_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655213912525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("longformer_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I am working at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("longformer_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I am working at John Snow Labs.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_15_new","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_15_new","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_scrambled_squad_15_new| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-15-new --- layout: model title: Fast Neural Machine Translation Model from English to Kwangali author: John Snow Labs name: opus_mt_en_kwn date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, kwn, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `kwn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_kwn_xx_2.7.0_2.4_1609164344098.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_kwn_xx_2.7.0_2.4_1609164344098.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_kwn| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Hindi Named Entity Recognition (from sagorsarker) author: John Snow Labs name: bert_ner_codeswitch_hineng_lid_lince date: 2022-05-09 tags: [bert, ner, token_classification, hi, open_source] task: Named Entity Recognition language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `codeswitch-hineng-lid-lince` is a Hindi model orginally trained by `sagorsarker`. ## Predicted Entities `mixed`, `hin`, `other`, `unk`, `en`, `ambiguous`, `ne`, `fw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_hineng_lid_lince_hi_3.4.2_3.0_1652097632881.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_hineng_lid_lince_hi_3.4.2_3.0_1652097632881.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_codeswitch_hineng_lid_lince| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|hi| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sagorsarker/codeswitch-hineng-lid-lince - https://ritual.uh.edu/lince/home - https://github.com/sagorbrur/codeswitch --- layout: model title: Pipeline to Detect Clinical Entities (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_jsl_pipeline date: 2023-03-20 tags: [ner_jsl, ner, berfortokenclassification, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_jsl](https://nlp.johnsnowlabs.com/2022/03/21/bert_token_classifier_ner_jsl_en_2_4.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.3.0_3.2_1679305183990.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.3.0_3.2_1679305183990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_token_ner_jsl.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:---------------------------------|--------:|------:|:-------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.999456 | | 1 | Caucasian male | 28 | 41 | Demographics | 0.9901 | | 2 | congestion | 62 | 71 | Symptom | 0.997918 | | 3 | mom | 75 | 77 | Demographics | 0.999013 | | 4 | yellow discharge | 99 | 114 | Symptom | 0.998663 | | 5 | nares | 135 | 139 | Body_Part | 0.998609 | | 6 | she | 147 | 149 | Demographics | 0.999442 | | 7 | mild problems with his breathing | 168 | 199 | Symptom | 0.930385 | | 8 | perioral cyanosis | 237 | 253 | Symptom | 0.99819 | | 9 | retractions | 258 | 268 | Symptom | 0.999783 | | 10 | One day ago | 272 | 282 | Date_Time | 0.999386 | | 11 | mom | 285 | 287 | Demographics | 0.999835 | | 12 | tactile temperature | 304 | 322 | Symptom | 0.999352 | | 13 | Tylenol | 345 | 351 | Drug | 0.999762 | | 14 | Baby-girl | 354 | 362 | Age | 0.980529 | | 15 | decreased p.o. intake | 382 | 402 | Symptom | 0.998978 | | 16 | His | 405 | 407 | Demographics | 0.999913 | | 17 | breast-feeding | 416 | 429 | Body_Part | 0.99954 | | 18 | his | 493 | 495 | Demographics | 0.999661 | | 19 | respiratory congestion | 497 | 518 | Symptom | 0.834984 | | 20 | He | 521 | 522 | Demographics | 0.999858 | | 21 | tired | 555 | 559 | Symptom | 0.999516 | | 22 | fussy | 574 | 578 | Symptom | 0.997592 | | 23 | over the past 2 days | 580 | 599 | Date_Time | 0.994786 | | 24 | albuterol | 642 | 650 | Drug | 0.999735 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_jsl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Chinese BertForQuestionAnswering model (from qalover) author: John Snow Labs name: bert_qa_chinese_pert_large_open_domain_mrc date: 2022-06-28 tags: [zh, open_source, bert, question_answering] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-pert-large-open-domain-mrc` is a Chinese model originally trained by `qalover`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_chinese_pert_large_open_domain_mrc_zh_4.0.0_3.0_1656413708959.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_chinese_pert_large_open_domain_mrc_zh_4.0.0_3.0_1656413708959.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_chinese_pert_large_open_domain_mrc","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_chinese_pert_large_open_domain_mrc","zh") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.large").predict("""PUT YOUR QUESTION HERE|||"PUT YOUR CONTEXT HERE""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_chinese_pert_large_open_domain_mrc| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/qalover/chinese-pert-large-open-domain-mrc - https://github.com/dbiir/UER-py/ --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes author: John Snow Labs name: sbiobertresolve_umls_major_concepts date: 2021-10-03 tags: [entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.2.3 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to 4 major categories of UMLS CUI codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It has faster load time, with a speedup of about 6X when compared to previous versions. ## Predicted Entities `This model returns CUI (concept unique identifier) codes for Clinical Findings`, `Medical Devices`, `Anatomical Structures and Injuries & Poisoning terms` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_major_concepts_en_3.2.3_3.0_1633221571574.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_major_concepts_en_3.2.3_3.0_1633221571574.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_umls_major_concepts``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Cerebrovascular_Disease, Communicable_Disease, Diabetes, Disease_Syndrome_Disorder, Heart_Disease, Hyperlipidemia, Hypertension, Injury_or_Poisoning, Kidney_Disease, Medical-Device, Obesity, Oncological, Overweight, Psychological_Condition, Symptom, VS_Finding, ImagingFindings, EKG_Findings``` set in ```.setWhiteList()```.

## Results ```bash | | ner_chunk | code | code_description | |---:|:------------------------------|:-------------|:---------------------------------------------| | 0 | ankle | C4047548 | bilateral ankle joint pain (finding) | | 1 | falling from stairs | C0417023 | fall from stairs | | 2 | Arthroscopy | C0179144 | arthroscope | | 3 | primary care pyhsician | C3266804 | referred by primary care physician (finding) | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_major_concepts| |Compatibility:|Healthcare NLP 3.2.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[umls_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on data sampled from https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: BioBERT Sentence Embeddings (PMC) author: John Snow Labs name: sent_biobert_pmc_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_pmc_base_cased_en_2.6.0_2.4_1598348966950.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_pmc_base_cased_en_2.6.0_2.4_1598348966950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_pmc_base_cased_embeddings I hate cancer [0.34035101532936096, 0.04413360357284546, -0.... Antibiotics aren't painkiller [0.4397204518318176, 0.066007100045681, -0.114... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: Explain Document Pipeline for Russian author: John Snow Labs name: explain_document_sm date: 2021-03-22 tags: [open_source, russian, explain_document_sm, pipeline, ru] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: ru edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_sm_ru_3.0.0_3.0_1616422668270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_sm_ru_3.0.0_3.0_1616422668270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_sm', lang = 'ru') annotations = pipeline.fullAnnotate(""Здравствуйте из Джона Снежных Лабораторий! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_sm", lang = "ru") val result = pipeline.fullAnnotate("Здравствуйте из Джона Снежных Лабораторий! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Здравствуйте из Джона Снежных Лабораторий! ""] result_df = nlu.load('ru.explain').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:------------------------------------------------|:-----------------------------------------------|:-----------------------------------------------------------|:-----------------------------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:-------------------------------| | 0 | ['Здравствуйте из Джона Снежных Лабораторий! '] | ['Здравствуйте из Джона Снежных Лабораторий!'] | ['Здравствуйте', 'из', 'Джона', 'Снежных', 'Лабораторий!'] | ['здравствовать', 'из', 'Джон', 'Снежных', 'Лабораторий!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['Джона Снежных Лабораторий!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|ru| --- layout: model title: Greek BertForMaskedLM Base Uncased model (from gealexandri) author: John Snow Labs name: bert_embeddings_greeksocial_base_greek_uncased_v1 date: 2022-12-06 tags: [el, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: el edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `greeksocialbert-base-greek-uncased-v1` is a Greek model originally trained by `gealexandri`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_greeksocial_base_greek_uncased_v1_el_4.2.4_3.0_1670326520370.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_greeksocial_base_greek_uncased_v1_el_4.2.4_3.0_1670326520370.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_greeksocial_base_greek_uncased_v1","el") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_greeksocial_base_greek_uncased_v1","el") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_greeksocial_base_greek_uncased_v1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|el| |Size:|424.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/gealexandri/greeksocialbert-base-greek-uncased-v1 - http://www.paloservices.com/ --- layout: model title: Contextual SpellChecker Clinical author: John Snow Labs name: spellcheck_clinical class: ContextSpellCheckerModel language: en nav_key: models repository: clinical/models date: 2020-04-17 task: Spell Check edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [clinical,licensed,en] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Implements Noisy Channel Model Spell Algorithm. Correction candidates are extracted combining context information and word information {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/6.Clinical_Context_Spell_Checker.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_en_2.4.2_2.4_1587146727460.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_en_2.4.2_2.4_1587146727460.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = ContextSpellCheckerModel.pretrained("spellcheck_clinical","en","clinical/models") .setInputCols("token") .setOutputCol("spell") ``` ```scala val model = ContextSpellCheckerModel.pretrained("spellcheck_clinical","en","clinical/models") .setInputCols("token") .setOutputCol("spell") ``` {:.nlu-block} ```python import nlu nlu.load("en.spell.clinical").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---------------|--------------------------| | Name: | spellcheck_clinical | | Type: | ContextSpellCheckerModel | | Compatibility: | 2.4.2 | | License: | Licensed | | Edition: | Official | |Input labels: | [token] | |Output labels: | [spell] | | Language: | en | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained with PubMed and i2b2 datasets. --- layout: model title: English BertForQuestionAnswering Base Cased model (from rsvp-ai) author: John Snow Labs name: bert_qa_bertserini_base_cmrc date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertserini-bert-base-cmrc` is a English model originally trained by `rsvp-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bertserini_base_cmrc_en_4.0.0_3.0_1657188963909.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bertserini_base_cmrc_en_4.0.0_3.0_1657188963909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bertserini_base_cmrc","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_bertserini_base_cmrc","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bertserini_base_cmrc| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|381.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rsvp-ai/bertserini-bert-base-cmrc --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from lingchensanwen) author: John Snow Labs name: distilbert_qa_lingchensanwen_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `lingchensanwen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_lingchensanwen_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771975946.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_lingchensanwen_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771975946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lingchensanwen_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lingchensanwen_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_lingchensanwen_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/lingchensanwen/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering model (from juliusco) author: John Snow Labs name: bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert-base-cased-v1.1-squad-finetuned-biobert` is a English model orginally trained by `juliusco`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert_en_4.0.0_3.0_1654185597741.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert_en_4.0.0_3.0_1654185597741.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.biobert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_base_cased_v1.1_squad_finetuned_biobert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/juliusco/biobert-base-cased-v1.1-squad-finetuned-biobert --- layout: model title: Clinical Deidentification (Spanish) author: John Snow Labs name: clinical_deidentification date: 2022-03-02 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with sciwiki_300d embeddings and can be used to deidentify PHI information from medical texts in Spanish. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask, fake or obfuscate the following entities: `AGE`, `DATE`, `PROFESSION`, `E-MAIL`, `USERNAME`, `LOCATION`, `DOCTOR`, `HOSPITAL`, `PATIENT`, `URL`, `IP`, `MEDICALRECORD`, `IDNUM`, `ORGANIZATION`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `PLATE`, `SEX` and `IPADDR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.4.1_2.4_1646246697330.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.4.1_2.4_1646246697330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from johnsnowlabs import * deid_pipeline = PretrainedPipeline("clinical_deidentification", "es", "clinical/models") sample = """Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com """ result = deid_pipeline .annotate(sample) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification", "es", "clinical/models") sample = "Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com " val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("es.deid.clinical").predict("""Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com """) ```

## Results ```bash Masked with entity labels ------------------------------ Datos del paciente. Nombre: . Apellidos: . NHC: . NASS: 04. Domicilio: , 5 B.. Localidad/ Provincia: . CP: . Datos asistenciales. Fecha de nacimiento: . País: . Edad: años Sexo: . Fecha de Ingreso: . : María Merino Viveros NºCol: . Informe clínico del paciente: de años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Servicio de Endocrinología y Nutrición km 12,500 28905 - () Correo electrónico: Masked with chars ------------------------------ Datos del paciente. Nombre: [**] . Apellidos: [*************]. NHC: [*****]. NASS: ** [******] 04. Domicilio: [*******************], 5 B.. Localidad/ Provincia: [****]. CP: [***]. Datos asistenciales. Fecha de nacimiento: [********]. País: [****]. Edad: ** años Sexo: *. Fecha de Ingreso: [********]. [****]: María Merino Viveros NºCol: ** ** [***]. Informe clínico del paciente: [***] de ** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en [*********] en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: [*************] +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente [****] significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. [******************] [******************************] Servicio de Endocrinología y Nutrición [*****************] km 12,500 28905 [****] - [****] ([****]) Correo electrónico: [********************] Masked with fixed length chars ------------------------------ Datos del paciente. Nombre: **** . Apellidos: ****. NHC: ****. NASS: **** **** 04. Domicilio: ****, 5 B.. Localidad/ Provincia: ****. CP: ****. Datos asistenciales. Fecha de nacimiento: ****. País: ****. Edad: **** años Sexo: ****. Fecha de Ingreso: ****. ****: María Merino Viveros NºCol: **** **** ****. Informe clínico del paciente: **** de **** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en **** en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: **** +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente **** significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. **** **** Servicio de Endocrinología y Nutrición **** km 12,500 28905 **** - **** (****) Correo electrónico: **** Obfuscated ------------------------------ Datos del paciente. Nombre: Sr. Lerma . Apellidos: Aristides Gonzalez Gelabert. NHC: BBBBBBBBQR648597. NASS: 041010000011 RZRM020101906017 04. Domicilio: Valencia, 5 B.. Localidad/ Provincia: Madrid. CP: 99335. Datos asistenciales. Fecha de nacimiento: 25/04/1977. País: Barcelona. Edad: 8 años Sexo: F.. Fecha de Ingreso: 02/08/2018. transportista: María Merino Viveros NºCol: olegario10 olegario10 felisa78. Informe clínico del paciente: RZRM020101906017 de 8 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Madrid en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Dra. Laguna +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente 041010000011 significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Reinaldo Manjón Malo Barcelona Servicio de Endocrinología y Nutrición Valencia km 12,500 28905 Bilbao - Madrid (Barcelona) Correo electrónico: quintanasalome@example.net ``` ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|281.2 MB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetectorDLModel - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - nlp.NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: Arabic Bert Embeddings (Base, Arabert Model, v01) author: John Snow Labs name: bert_embeddings_bert_base_arabertv01 date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabertv01` is a Arabic model orginally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabertv01_ar_3.4.2_3.0_1649677579686.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabertv01_ar_3.4.2_3.0_1649677579686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabertv01| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|508.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-base-arabertv01 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-base-arabertv01/resolve/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: English RobertaForQuestionAnswering (from mbartolo) author: John Snow Labs name: roberta_qa_roberta_large_synqa_ext date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-synqa-ext` is a English model originally trained by `mbartolo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_synqa_ext_en_4.0.0_3.0_1655738082187.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_synqa_ext_en_4.0.0_3.0_1655738082187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_synqa_ext","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_synqa_ext","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.synqa_ext.roberta.large.by_mbartolo").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_synqa_ext| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mbartolo/roberta-large-synqa-ext - https://arxiv.org/abs/2002.00293 - https://arxiv.org/abs/2104.08678 --- layout: model title: Part of Speech for Latin author: John Snow Labs name: pos_ud_llct date: 2021-03-09 tags: [part_of_speech, open_source, latin, pos_ud_llct, la] task: Part of Speech Tagging language: la edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - PUNCT - ADP - PROPN - NOUN - VERB - DET - CCONJ - PRON - ADJ - NUM - AUX - SCONJ - ADV - PART - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_llct_la_3.0.0_3.0_1615292206384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_llct_la_3.0.0_3.0_1615292206384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_llct", "la") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Aequaliter Nubila Labs Ioannes de salve ! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_llct", "la") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Aequaliter Nubila Labs Ioannes de salve ! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Aequaliter Nubila Labs Ioannes de salve ! ""] token_df = nlu.load('la.pos').predict(text) token_df ```

## Results ```bash token pos 0 Aequaliter PROPN 1 Nubila PROPN 2 Labs ADJ 3 Ioannes NOUN 4 de ADP 5 salve NOUN 6 ! PROPN ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_llct| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|la| --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-10` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10_en_4.0.0_3.0_1654189512725.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10_en_4.0.0_3.0_1654189512725.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_1024d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|390.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-10 --- layout: model title: Word2Vec Embeddings in Tatar (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, tt, open_source] task: Embeddings language: tt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tt_3.4.1_3.0_1647462888271.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tt_3.4.1_3.0_1647462888271.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_base_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_base_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_embeddings I [0.5920650362968445, 0.18827693164348602, 0.12... love [1.2889715433120728, 0.8475795388221741, 0.720... NLP [0.21503107249736786, -0.9925870299339294, 1.0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1](https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1) --- layout: model title: Czech RobertaForMaskedLM Cased model (from fav-kky) author: John Snow Labs name: roberta_embeddings_fernet_news date: 2022-12-12 tags: [cs, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: cs edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `FERNET-News` is a Czech model originally trained by `fav-kky`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_fernet_news_cs_4.2.4_3.0_1670858382244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_fernet_news_cs_4.2.4_3.0_1670858382244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_fernet_news","cs") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_fernet_news","cs") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_fernet_news| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|cs| |Size:|468.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/fav-kky/FERNET-News - https://arxiv.org/abs/2107.10042 --- layout: model title: English BertForQuestionAnswering model (from lewtun) author: John Snow Labs name: bert_qa_bert_base_uncased_finetuned_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad-v1` is a English model orginally trained by `lewtun`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_squad_v1_en_4.0.0_3.0_1654181199655.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_squad_v1_en_4.0.0_3.0_1654181199655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_finetuned_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_finetuned_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_lewtun").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_finetuned_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/lewtun/bert-base-uncased-finetuned-squad-v1 --- layout: model title: English Named Entity Recognition (from lucifermorninstar011) author: John Snow Labs name: distilbert_ner_autotrain_luicfer_company_861827409 date: 2022-05-16 tags: [distilbert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-luicfer_company-861827409` is a English model orginally trained by `lucifermorninstar011`. ## Predicted Entities `vocab`, `company` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_autotrain_luicfer_company_861827409_en_3.4.2_3.0_1652721660864.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_autotrain_luicfer_company_861827409_en_3.4.2_3.0_1652721660864.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_autotrain_luicfer_company_861827409| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/lucifermorninstar011/autotrain-luicfer_company-861827409 --- layout: model title: Summarize clinical notes (augmented) author: John Snow Labs name: summarizer_clinical_jsl_augmented date: 2023-03-30 tags: [licensed, clinical, en, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based summarization model that is at first finetuned with natural instructions and then finetuned with clinical notes, encounters, critical care notes, discharge notes, reports, curated by John Snow Labs. This model is further optimized by augmenting the training methodology, and dataset. It can generate summaries from clinical notes up to 512 tokens given the input text (max 1024 tokens). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/MEDICAL_TEXT_SUMMARIZATION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/32.Medical_Text_Summarization.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_augmented_en_4.3.2_3.0_1680203312371.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_augmented_en_4.3.2_3.0_1680203312371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") summarizer = MedicalSummarizer()\ .pretrained("summarizer_clinical_jsl_augmented", "en", "clinical/models")\ .setInputCols("document")\ .setOutputCol("summary")\ .setMaxTextLength(512)\ .setMaxNewTokens(512) pipeline = Pipeline(stages=[document, summarizer]) text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val summarizer = MedicalSummarizer() .pretrained("summarizer_clinical_jsl_augmented", "en", "clinical/models") .setInputCols("document") .setOutputCol("summary") .setMaxTextLength(512) .setMaxNewTokens(512) val pipeline = new Pipeline().setStages(Array(document, summarizer)) val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" val data = Seq(text).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash A 78-year-old female with hypertension, syncope, and spinal stenosis returns for a recheck. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. Her medications include Atenolol, Premarin, calcium with vitamin D, multivitamin, aspirin, and TriViFlor. She also has Elocon cream and Synalar cream for rash. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_jsl_augmented| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.0 MB| ## Benchmarkings ### Benchmark on MtSamples Summarization Dataset : | model_name | model_size | rouge | bleu | bertscore_precision | bertscore_recall: | bertscore_f1 | |--|--|--|--|--|--|--| philschmid/flan-t5-base-samsum | 250M | 0.1919 | 0.1124 | 0.8409 | 0.8964 | 0.8678 | linydub/bart-large-samsum | 500M | 0.1586 | 0.0732 | 0.8747 | 0.8184 | 0.8456 | philschmid/bart-large-cnn-samsum | 500M | 0.2170 | 0.1299 | 0.8846 | 0.8436 | 0.8636 | transformersbook/pegasus-samsum | 500M | 0.1924 | 0.0965 | 0.8920 | 0.8149 | 0.8517 | summarizer_clinical_jsl | 250M | 0.4836 | 0.4188 | 0.9041 | 0.9374 | 0.9204 | summarizer_clinical_jsl_augmented | 250M | 0.5119 | 0.4545 | 0.9282 | 0.9526 | 0.9402 | ### Benchmark on MIMIC Summarization Dataset : | model_name | model_size | rouge | bleu | bertscore_precision | bertscore_recall: | bertscore_f1 | |--|--|--|--|--|--|--| philschmid/flan-t5-base-samsum | 250M | 0.1910 | 0.1037 | 0.8708 | 0.9056 | 0.8879 | linydub/bart-large-samsum | 500M | 0.1252 | 0.0382 | 0.8933 | 0.8440 | 0.8679 | philschmid/bart-large-cnn-samsum | 500M | 0.1795 | 0.0889 | 0.9172 | 0.8978 | 0.9074 | transformersbook/pegasus-samsum | 570M | 0.1425 | 0.0582 | 0.9171 | 0.8682 | 0.8920 | summarizer_clinical_jsl | 250M | 0.395 | 0.2962 | 0.895 | 0.9316 | 0.913 | summarizer_clinical_jsl_augmented | 250M | 0.3964 | 0.307 | 0.9109 | 0.9452 | 0.9227 | ![Benchmark Summary](https://github.com/JohnSnowLabs/jsl-private-projects/blob/llm_benchmarks/internal_projects/LLM_Experiments/jsl-summarization-benchmarks.png?raw=true) ## References Trained on in-house curated dataset --- layout: model title: Part of Speech for Norwegian Nynorsk author: John Snow Labs name: pos_ud_nynorsk date: 2020-05-05 18:57:00 +0800 task: Part of Speech Tagging language: nn edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, nn] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_nynorsk_nn_2.5.0_2.4_1588693690964.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_nynorsk_nn_2.5.0_2.4_1588693690964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_nynorsk", "nn") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_nynorsk", "nn") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene."""] pos_df = nlu.load('nn.pos.ud_nynorsk').predict(text) pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='NOUN', metadata={'word': 'Annet'}), Row(annotatorType='pos', begin=6, end=8, result='SCONJ', metadata={'word': 'enn'}), Row(annotatorType='pos', begin=10, end=10, result='PART', metadata={'word': 'å'}), Row(annotatorType='pos', begin=12, end=15, result='VERB', metadata={'word': 'være'}), Row(annotatorType='pos', begin=17, end=22, result='NOUN', metadata={'word': 'kongen'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_nynorsk| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|nn| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Chinese Bert Embeddings (Large, MacBERT) author: John Snow Labs name: bert_embeddings_chinese_macbert_large date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-macbert-large` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_macbert_large_zh_3.4.2_3.0_1649669165054.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_macbert_large_zh_3.4.2_3.0_1649669165054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_macbert_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-macbert-large - https://github.com/ymcui/MacBERT/blob/master/LICENSE - https://2020.emnlp.org - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://github.com/chatopera/Synonyms - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 --- layout: model title: Mapping Drug Brand Names with Corresponding National Drug Codes author: John Snow Labs name: drug_brandname_ndc_mapper date: 2022-05-11 tags: [chunk_mapper, en, licensed, ndc, clinical] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps drug brand names to corresponding National Drug Codes (NDC). Product NDCs for each strength are returned in result and metadata. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/drug_brandname_ndc_mapper_en_3.5.1_3.0_1652259542096.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/drug_brandname_ndc_mapper_en_3.5.1_3.0_1652259542096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("chunk") chunkerMapper = ChunkMapperModel.pretrained("drug_brandname_ndc_mapper", "en", "clinical/models")\ .setInputCols(["chunk"])\ .setOutputCol("ndc")\ .setRel("Strength_NDC") pipeline = Pipeline().setStages([document_assembler, chunkerMapper]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) result = lp.fullAnnotate(["zytiga", "zyvana", "ZYVOX", "ZYTIGA"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("chunk") val chunkerMapper = ChunkMapperModel.pretrained("drug_brandname_ndc_mapper", "en", "clinical/models") .setInputCols("chunk") .setOutputCol("ndc") .setRel("Strength_NDC") val pipeline = new Pipeline().setStages(Array(document_assembler, chunkerMapper)) val text_data = Seq("zytiga", "zyvana", "ZYVOX", "ZYTIGA").toDS.toDF("text") val res = pipeline.fit(text_data).transform(text_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.drug_brand_to_ndc").predict("""Put your text here.""") ```

## Results ```bash |---:|:------------|:-------------------------|:----------------------------------------------------------| | | Brandname | Strenth_NDC | Other_NDSs | |---:|:------------|:-------------------------|:----------------------------------------------------------| | 0 | zytiga | 500 mg/1 | 57894-195 | ['250 mg/1 | 57894-150'] | | 1 | zyvana | 527 mg/1 | 69336-405 | [''] | | 2 | ZYVOX | 600 mg/300mL | 0009-4992 | ['600 mg/300mL | 66298-7807', '600 mg/300mL | 0009-7807'] | | 3 | ZYTIGA | 500 mg/1 | 57894-195 | ['250 mg/1 | 57894-150'] | |---:|:------------|:-------------------------|:----------------------------------------------------------| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|drug_brandname_ndc_mapper| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|3.0 MB| --- layout: model title: English image_classifier_vit_rock_challenge_DeiT_solo ViTForImageClassification from dimbyTa author: John Snow Labs name: image_classifier_vit_rock_challenge_DeiT_solo date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rock_challenge_DeiT_solo` is a English model originally trained by dimbyTa. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rock_challenge_DeiT_solo_en_4.1.0_3.0_1660170757484.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rock_challenge_DeiT_solo_en_4.1.0_3.0_1660170757484.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rock_challenge_DeiT_solo", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rock_challenge_DeiT_solo", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rock_challenge_DeiT_solo| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|81.7 MB| --- layout: model title: Pipeline to Detect Bacterial Species (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_bacteria_pipeline date: 2023-03-20 tags: [bacteria, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_bacteria](https://nlp.johnsnowlabs.com/2022/01/07/bert_token_classifier_ner_bacteria_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_pipeline_en_4.3.0_3.2_1679305685030.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_pipeline_en_4.3.0_3.2_1679305685030.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_bacteria_pipeline", "en", "clinical/models") text = '''Based on these genetic and phenotypic properties, we propose that strain SMSP (T) represents a novel species of the genus Methanoregula, for which we propose the name Methanoregula formicica sp. nov., with the type strain SMSP (T) (= NBRC 105244 (T) = DSM 22288 (T)).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_bacteria_pipeline", "en", "clinical/models") val text = "Based on these genetic and phenotypic properties, we propose that strain SMSP (T) represents a novel species of the genus Methanoregula, for which we propose the name Methanoregula formicica sp. nov., with the type strain SMSP (T) (= NBRC 105244 (T) = DSM 22288 (T))." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.bacteria_ner.pipeline").predict("""Based on these genetic and phenotypic properties, we propose that strain SMSP (T) represents a novel species of the genus Methanoregula, for which we propose the name Methanoregula formicica sp. nov., with the type strain SMSP (T) (= NBRC 105244 (T) = DSM 22288 (T)).""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------|--------:|------:|:------------|-------------:| | 0 | SMSP (T) | 73 | 80 | SPECIES | 0.99985 | | 1 | Methanoregula formicica | 167 | 189 | SPECIES | 0.999787 | | 2 | SMSP (T) | 222 | 229 | SPECIES | 0.999871 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bacteria_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Turkish Bert Embeddings author: John Snow Labs name: bert_embeddings_bert_base_tr_cased date: 2022-04-11 tags: [bert, embeddings, tr, open_source] task: Embeddings language: tr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-tr-cased` is a Turkish model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_tr_cased_tr_3.4.2_3.0_1649675409597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_tr_cased_tr_3.4.2_3.0_1649675409597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_tr_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|tr| |Size:|378.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-tr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Zero-Shot Named Entity Recognition (RoBERTa) author: John Snow Labs name: zero_shot_ner_roberta date: 2022-08-29 tags: [ner, zero_shot, licensed, clinical, en, roberta] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: ZeroShotNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with Zero-Shot Named Entity Recognition (NER) approach and it can detect any kind of defined entities with no training dataset, just pretrained RoBERTa embeddings (included in the model). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/zero_shot_ner_roberta_en_4.0.2_3.0_1661769801401.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/zero_shot_ner_roberta_en_4.0.2_3.0_1661769801401.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+---------+--------+-----+---+----------+ | token|ner_label|sentence|begin|end|confidence| +------+---------+--------+-----+---+----------+ |Hellen| B-NAME| 0| 0| 5|0.13306311| | works| O| 0| 7| 11| null| | in| O| 0| 13| 14| null| |London| B-CITY| 0| 16| 21| 0.4064213| | ,| O| 0| 22| 22| null| | Paris| B-CITY| 0| 24| 28|0.04597357| | and| O| 0| 30| 32| null| |Berlin| B-CITY| 0| 34| 39|0.16265489| | .| O| 0| 40| 40| null| | My| O| 1| 42| 43| null| | name| O| 1| 45| 48| null| | is| O| 1| 50| 51| null| | Clara| B-NAME| 1| 53| 57| 0.9274031| | ,| O| 1| 58| 58| null| | I| O| 1| 60| 60| null| | live| O| 1| 62| 65| null| | in| O| 1| 67| 68| null| | New| B-CITY| 1| 70| 72|0.82799006| | York| I-CITY| 1| 74| 77|0.82799006| | and| O| 1| 79| 81| null| |Hellen| B-NAME| 1| 83| 88|0.40429682| | lives| O| 1| 90| 94| null| | in| O| 1| 96| 97| null| | Paris| B-CITY| 1| 99|103|0.49216735| | .| O| 1| 104|104| null| | John| B-NAME| 0| 0| 3|0.14063153| | is| O| 0| 5| 6| null| | a| O| 0| 8| 8| null| | man| O| 0| 10| 12| null| | who| O| 0| 14| 16| null| | works| O| 0| 18| 22| null| | in| O| 0| 24| 25| null| |London| B-CITY| 0| 27| 32|0.15521188| | ,| O| 0| 33| 33| null| |London| B-CITY| 0| 35| 40|0.12151082| | and| O| 0| 42| 44| null| |London| B-CITY| 0| 46| 51| 0.2650951| | .| O| 0| 52| 52| null| +------+---------+--------+-----+---+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|zero_shot_ner_roberta| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References As it is a Zero-Shot NER, no training dataset is necessary. --- layout: model title: Fast Neural Machine Translation Model from Arabic to Italian author: John Snow Labs name: opus_mt_ar_it date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, it, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: it {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_it_xx_3.1.0_2.4_1622556125806.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_it_xx_3.1.0_2.4_1622556125806.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_it| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from deepset) author: John Snow Labs name: roberta_qa_deepset_large_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad2` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_large_squad2_en_4.2.4_3.0_1669987928587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_large_squad2_en_4.2.4_3.0_1669987928587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_large_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_large_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_large_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/deepset/roberta-large-squad2 --- layout: model title: Legal Question Answering (RoBerta, CUAD, Base) author: John Snow Labs name: legqa_roberta_cuad_base date: 2023-01-30 tags: [en, licensed, tensorflow] task: Question Answering language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Legal RoBerta-based Question Answering model, trained on squad-v2, finetuned on CUAD dataset (base). In order to use it, a specific prompt is required. This is an example of it for extracting PARTIES: ``` "Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract" ``` ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legqa_roberta_cuad_base_en_1.0.0_3.0_1675083334950.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legqa_roberta_cuad_base_en_1.0.0_3.0_1675083334950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) spanClassifier = nlp.RoBertaForQuestionAnswering.pretrained("legqa_roberta_cuad_base","en", "legal/models") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = nlp.Pipeline().setStages([ documentAssembler, spanClassifier ]) text = """THIS CREDIT AGREEMENT is dated as of April 29, 2010, and is made by and among P.H. GLATFELTER COMPANY, a Pennsylvania corporation ( the "COMPANY") and certain of its subsidiaries. Identified on the signature pages hereto (each a "BORROWER" and collectively, the "BORROWERS"), each of the GUARANTORS (as hereinafter defined), the LENDERS (as hereinafter defined), PNC BANK, NATIONAL ASSOCIATION, in its capacity as agent for the Lenders under this Agreement (hereinafter referred to in such capacity as the "ADMINISTRATIVE AGENT"), and, for the limited purpose of public identification in trade tables, PNC CAPITAL MARKETS LLC and CITIZENS BANK OF PENNSYLVANIA, as joint arrangers and joint bookrunners, and CITIZENS BANK OF PENNSYLVANIA, as syndication agent.""".replace('\n',' ') question = ['"Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract"'] qt = [ [q,text] for q in questions ] example = spark.createDataFrame(qt).toDF("question", "context") result = pipeline.fit(example).transform(example) result.select('document_question.result', 'answer.result').show(truncate=False) ```

## Results ```bash ["Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract"]|[P . H . GLATFELTER COMPANY]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legqa_roberta_cuad_base| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|453.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References Squad, finetuned with CUAD-based Question/Answering --- layout: model title: English Named Entity Recognition (from DeDeckerThomas) author: John Snow Labs name: distilbert_ner_keyphrase_extraction_distilbert_kptimes date: 2022-05-16 tags: [distilbert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyphrase-extraction-distilbert-kptimes` is a English model orginally trained by `DeDeckerThomas`. ## Predicted Entities `KEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_keyphrase_extraction_distilbert_kptimes_en_3.4.2_3.0_1652721921747.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_keyphrase_extraction_distilbert_kptimes_en_3.4.2_3.0_1652721921747.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_keyphrase_extraction_distilbert_kptimes| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/DeDeckerThomas/keyphrase-extraction-distilbert-kptimes - https://paperswithcode.com/sota?task=Keyphrase+Extraction&dataset=kptimes --- layout: model title: Multilingual T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_flan_small date: 2023-01-30 tags: [vi, ne, fi, ur, ku, yo, si, ru, it, zh, la, hi, he, xh, so, ca, ar, as, sw, en, ro, ig, te, th, ta, ce, es, gu, or, fr, ka, "no", li, cr, ch, be, ha, ga, ja, pa, ko, sl, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `flan-t5-small` is a Multilingual model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_flan_small_xx_4.3.0_3.0_1675102370004.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_flan_small_xx_4.3.0_3.0_1675102370004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_flan_small","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_flan_small","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_flan_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|349.5 MB| ## References - https://huggingface.co/google/flan-t5-small - https://s3.amazonaws.com/moonup/production/uploads/1666363435475-62441d1d9fdefb55a0b7d12c.png - https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints - https://arxiv.org/pdf/2210.11416.pdf - https://github.com/google-research/t5x - https://arxiv.org/pdf/2210.11416.pdf - https://arxiv.org/pdf/2210.11416.pdf - https://arxiv.org/pdf/2210.11416.pdf - https://s3.amazonaws.com/moonup/production/uploads/1666363265279-62441d1d9fdefb55a0b7d12c.png - https://arxiv.org/pdf/2210.11416.pdf - https://github.com/google-research/t5x - https://github.com/google/jax - https://s3.amazonaws.com/moonup/production/uploads/1668072995230-62441d1d9fdefb55a0b7d12c.png - https://arxiv.org/pdf/2210.11416.pdf - https://arxiv.org/pdf/2210.11416.pdf - https://mlco2.github.io/impact#compute - https://arxiv.org/abs/1910.09700 --- layout: model title: Multilingual XLMRoBerta Embeddings author: John Snow Labs name: xlmroberta_embeddings_afriberta_base date: 2022-05-13 tags: [ha, yo, ig, am, so, open_source, xlm_roberta, embeddings, xx, afriberta] task: Embeddings language: xx edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true recommended: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `afriberta_base` is a Multilingual model orginally trained by `castorini`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_afriberta_base_xx_3.4.4_3.0_1652439193066.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_afriberta_base_xx_3.4.4_3.0_1652439193066.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_afriberta_base| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|417.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/castorini/afriberta_base - https://github.com/keleog/afriberta --- layout: model title: English asr_wav2vec2_med_custom_train_large TFWav2Vec2ForCTC from PrajwalS author: John Snow Labs name: asr_wav2vec2_med_custom_train_large date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_med_custom_train_large` is a English model originally trained by PrajwalS. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_med_custom_train_large_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_med_custom_train_large_en_4.2.0_3.0_1664122216388.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_med_custom_train_large_en_4.2.0_3.0_1664122216388.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_med_custom_train_large", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_med_custom_train_large", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_med_custom_train_large| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Malay T5ForConditionalGeneration Tiny Cased model (from mesolitica) author: John Snow Labs name: t5_tiny_bahasa_cased date: 2023-01-31 tags: [ms, open_source, t5, tensorflow] task: Text Generation language: ms edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-tiny-bahasa-cased` is a Malay model originally trained by `mesolitica`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_tiny_bahasa_cased_ms_4.3.0_3.0_1675156097275.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_tiny_bahasa_cased_ms_4.3.0_3.0_1675156097275.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_tiny_bahasa_cased","ms") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_tiny_bahasa_cased","ms") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_tiny_bahasa_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ms| |Size:|90.5 MB| ## References - https://huggingface.co/mesolitica/t5-tiny-bahasa-cased - https://github.com/huseinzol05/malaya/tree/master/pretrained-model/t5/prepare - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5 --- layout: model title: Legal Relation Extraction (Parties, Alias, Dates, Document Type, Sm, Bidirectional) author: John Snow Labs name: legre_contract_doc_parties date: 2022-08-12 tags: [en, legal, re, relations, agreements, licensed] task: Relation Extraction language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_introduction_clause` Text Classifier to select only these paragraphs; This is a Legal Relation Extraction model, which can be used after the NER Model for extracting Parties, Document Types, Effective Dates and Aliases, called `legner_contract_doc_parties`. As an output, you will get the relations linking the different concepts together, if such relation exists. The list of relations is: - dated_as: A Document has an Effective Date - has_alias: The Alias of a Party all along the document - has_collective_alias: An Alias hold by several parties at the same time - signed_by: Between a Party and the document they signed This model is a `sm` model without meaningful directions in the relations (the model was not trained to understand if the direction of the relation is from left to right or right to left). There are bigger models in Models Hub trained also with directed relationships. ## Predicted Entities `dated_as`, `has_alias`, `has_collective_alias`, `signed_by` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/LEGALRE_PARTIES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legre_contract_doc_parties_en_1.0.0_3.2_1660293010932.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legre_contract_doc_parties_en_1.0.0_3.2_1660293010932.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_base_uncased_legal", "en") \ .setInputCols("document", "token") \ .setOutputCol("embeddings") ner_model = legal.NerModel.pretrained('legner_contract_doc_parties', 'en', 'legal/models')\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") reDL = legal.RelationExtractionDLModel().pretrained('legre_contract_doc_parties', 'en', 'legal/models')\ .setPredictionThreshold(0.5)\ .setInputCols(["ner_chunk", "document"])\ .setOutputCol("relations") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner_model, ner_converter, reDL ]) text=''' This INTELLECTUAL PROPERTY AGREEMENT (this "Agreement"), dated as of December 31, 2018 (the "Effective Date") is entered into by and between Armstrong Flooring, Inc., a Delaware corporation ("Seller") and AFI Licensing LLC, a Delaware limited liability company ("Licensing" and together with Seller, "Arizona") and AHF Holding, Inc. (formerly known as Tarzan HoldCo, Inc.), a Delaware corporation ("Buyer") and Armstrong Hardwood Flooring Company, a Tennessee corporation (the "Company" and together with Buyer the "Buyer Entities") (each of Arizona on the one hand and the Buyer Entities on the other hand, a "Party" and collectively, the "Parties"). ''' data = spark.createDataFrame([[text]]).toDF("text") model = nlpPipeline.fit(data) ```

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence dated_as DOC 6 36 INTELLECTUAL PROPERTY AGREEMENT EFFDATE 70 86 December 31, 2018 0.9933402 signed_by DOC 6 36 INTELLECTUAL PROPERTY AGREEMENT PARTY 142 164 Armstrong Flooring, Inc 0.6235637 signed_by DOC 6 36 INTELLECTUAL PROPERTY AGREEMENT PARTY 316 331 AHF Holding, Inc 0.5001139 has_alias PARTY 142 164 Armstrong Flooring, Inc ALIAS 193 198 Seller 0.93385726 has_alias PARTY 206 222 AFI Licensing LLC ALIAS 264 272 Licensing 0.9859913 has_collective_alias ALIAS 293 298 Seller ALIAS 302 308 Arizona 0.82137156 has_alias PARTY 316 331 AHF Holding, Inc ALIAS 400 404 Buyer 0.8178999 has_alias PARTY 412 446 Armstrong Hardwood Flooring Company ALIAS 479 485 Company 0.9557921 has_alias PARTY 412 446 Armstrong Hardwood Flooring Company ALIAS 575 579 Buyer 0.6778585 has_alias PARTY 412 446 Armstrong Hardwood Flooring Company ALIAS 612 616 Party 0.6778583 has_alias PARTY 412 446 Armstrong Hardwood Flooring Company ALIAS 642 648 Parties 0.6778585 has_collective_alias ALIAS 506 510 Buyer ALIAS 517 530 Buyer Entities 0.69863707 has_collective_alias ALIAS 517 530 Buyer Entities ALIAS 575 579 Buyer 0.55453944 has_collective_alias ALIAS 517 530 Buyer Entities ALIAS 612 616 Party 0.55453944 has_collective_alias ALIAS 517 530 Buyer Entities ALIAS 642 648 Parties 0.55453944 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legre_contract_doc_parties| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|409.9 MB| ## References Manual annotations on CUAD dataset ## Benchmarking ```bash label Recall Precision F1 Support dated_as 0.962 0.962 0.962 26 has_alias 0.936 0.946 0.941 94 has_collective_alias 1.000 1.000 1.000 7 no_rel 0.982 0.980 0.981 497 signed_by 0.961 0.961 0.961 76 Avg. 0.968 0.970 0.969 - Weighted-Avg. 0.973 0.973 0.973 - ``` --- layout: model title: Hocr for table recognition pdf author: John Snow Labs name: hocr_table_recognition_pdf date: 2023-01-23 tags: [en, licensed] task: HOCR Table Recognition language: en nav_key: models edition: Visual NLP 4.2.4 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Table structure recognition based on hocr with Tesseract architecture, for PDF documents. Tesseract has been trained on a variety of datasets to improve its recognition capabilities. These datasets include images of text in various languages and scripts, as well as images with different font styles, sizes, and orientations. The training process involves feeding the engine with a large number of images and their corresponding text, allowing the engine to learn the patterns and characteristics of different text styles. One of the most important datasets used in training Tesseract is the UNLV dataset, which contains over 400,000 images of text in different languages, scripts, and font styles. This dataset is widely used in the OCR community and has been instrumental in improving the accuracy of Tesseract. Other datasets that have been used in training Tesseract include the ICDAR dataset, the IIIT-HWS dataset, and the RRC-GV-WS dataset. In addition to these datasets, Tesseract also uses a technique called adaptive training, where the engine can continuously improve its recognition capabilities by learning from new images and text. This allows Tesseract to adapt to new text styles and languages, and improve its overall accuracy. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/PDF_TABLE_RECOGNITION_HOCR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/tree/master/jupyter/SparkOCRPdfToTable.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pdf_to_hocr = PdfToHocr() \ .setInputCol("content") \ .setOutputCol("hocr") tokenizer = HocrTokenizer() \ .setInputCol("hocr") \ .setOutputCol("token") \ pdf_to_image = PdfToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setPageNumCol("tmp_pagenum") \ .setImageType(ImageType.TYPE_3BYTE_BGR) table_detector = ImageTableDetector \ .pretrained("general_model_table_detection_v2", "en", "public/ocr/models") \ .setInputCol("image") \ .setOutputCol("table_regions") \ .setScoreThreshold(0.9) \ .setApplyCorrection(True) \ .setScaleWidthToCol("width_dimension") \ .setScaleHeightToCol("height_dimension") image_scaler = ImageScaler() \ .setWidthCol("width_dimension") \ .setHeightCol("height_dimension") hocr_to_table = HocrToTextTable() \ .setInputCol("hocr") \ .setRegionCol("table_regions") \ .setOutputCol("tables") draw_annotations = ImageDrawAnnotations() \ .setInputCol("scaled_image") \ .setInputChunksCol("tables") \ .setOutputCol("image_with_annotations") \ .setFilledRect(False) \ .setFontSize(5) \ .setRectColor(Color.red) draw_regions = ImageDrawRegions() \ .setInputCol("scaled_image") \ .setInputRegionsCol("table_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.red) pipeline1 = PipelineModel(stages=[ pdf_to_hocr, tokenizer, pdf_to_image, table_detector, image_scaler, draw_regions, hocr_to_table ]) test_image_path = "data/pdfs/f1120.pdf" bin_df = spark.read.format("binaryFile").load(test_image_path) result = pipeline1.transform(bin_df).cache().drop("tmp_pagenum") result = result.filter(result.pagenum == 1) ``` ```scala val pdf_to_hocr = new PdfToHocr() .setInputCol("content") .setOutputCol("hocr") val tokenizer = new HocrTokenizer() .setInputCol("hocr") .setOutputCol("token") val pdf_to_image = new PdfToImage() .setInputCol("content") .setOutputCol("image") .setPageNumCol("tmp_pagenum") .setImageType(ImageType.TYPE_3BYTE_BGR) val table_detector = ImageTableDetector .pretrained("general_model_table_detection_v2", "en", "public/ocr/models") .setInputCol("image") .setOutputCol("table_regions") .setScoreThreshold(0.9) .setApplyCorrection(True) .setScaleWidthToCol("width_dimension") .setScaleHeightToCol("height_dimension") val image_scaler = new ImageScaler() .setWidthCol("width_dimension") .setHeightCol("height_dimension") val hocr_to_table = new HocrToTextTable() .setInputCol("hocr") .setRegionCol("table_regions") .setOutputCol("tables") val draw_annotations = new ImageDrawAnnotations() .setInputCol("scaled_image") .setInputChunksCol("tables") .setOutputCol("image_with_annotations") .setFilledRect(False) .setFontSize(5) .setRectColor(Color.red) val draw_regions = new ImageDrawRegions() .setInputCol("scaled_image") .setInputRegionsCol("table_regions") .setOutputCol("image_with_regions") .setRectColor(Color.red) val pipeline1 = new PipelineModel().setStages(Array( pdf_to_hocr, tokenizer, pdf_to_image, table_detector, image_scaler, draw_regions, hocr_to_table)) val test_image_path = "data/pdfs/f1120.pdf" val bin_df = spark.read.format("binaryFile").load(test_image_path) val result = pipeline1.transform(bin_df).cache().drop("tmp_pagenum") result = result.filter(result.pagenum == 1) ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image14.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image14_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash path modificationTime length hocr height_dimension width_dimension pagenum token image total_pages documentnum table_regions scaled_image image_with_regions tables exception table_index file:/content/f11... 2023-01-23 08:16:... 3471478

Live Demo [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_RoBERTa_hindi_guj_san_hi_3.4.2_3.0_1649947496602.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_RoBERTa_hindi_guj_san_hi_3.4.2_3.0_1649947496602.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_RoBERTa_hindi_guj_san| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|252.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/surajp/RoBERTa-hindi-guj-san - https://github.com/goru001/inltk - https://www.kaggle.com/disisbig/hindi-wikipedia-articles-172k - https://www.kaggle.com/disisbig/gujarati-wikipedia-articles - https://www.kaggle.com/disisbig/sanskrit-wikipedia-articles - https://twitter.com/parmarsuraj99 - https://www.linkedin.com/in/parmarsuraj99/ --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from iis2009002) author: John Snow Labs name: xlmroberta_ner_iis2009002_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `iis2009002`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_iis2009002_base_finetuned_panx_all_xx_4.1.0_3.0_1660428464219.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_iis2009002_base_finetuned_panx_all_xx_4.1.0_3.0_1660428464219.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_iis2009002_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_iis2009002_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_iis2009002_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/iis2009002/xlm-roberta-base-finetuned-panx-all --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_Modified_biobert_v1.1 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4_Modified-biobert-v1.1` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Modified_biobert_v1.1_en_4.0.0_3.0_1657109068197.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Modified_biobert_v1.1_en_4.0.0_3.0_1657109068197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_Modified_biobert_v1.1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4_Modified-biobert-v1.1 --- layout: model title: Financial Relation Extraction (Work Experience, Medium, Unidirectional) author: John Snow Labs name: finre_work_experience_md date: 2022-11-08 tags: [work, experience, role, en, licensed] task: Relation Extraction language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole financial report. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `finclf_work_experience_item` Text Classifier to select only these paragraphs; This is a `md` (medium) version of `finre_work_experience` model, trained with more data and with **unidirectional relation extractions**, meaning now the direction of the arrow matters: it goes from the source (`chunk1`) to the target (`chunk2`). This model allows you to analyzed present and past job positions of people, extracting relations between PERSON, ORG, ROLE and DATE. This model requires an NER with the mentioned entities, as `finner_org_per_role_date` and can also be combined with `finassertiondl_past_roles` to detect if the entities are mentioned to have happened in the PAST or not (although you can also infer that from the relations as `had_role_until`). ## Predicted Entities `has_role`, `had_role_until`, `has_role_from`, `works_for`, `has_role_in_company` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finre_work_experience_md_en_1.0.0_3.0_1667922980930.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finre_work_experience_md_en_1.0.0_3.0_1667922980930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence has_role_from DATE 3 19 December 15, 2021 ROLE 57 65 President 0.9532135 has_role_from DATE 3 19 December 15, 2021 ROLE 71 93 Chief Executive Officer 0.91833746 has_role PERSON 22 35 Anirudh Devgan ROLE 57 65 President 0.9993814 has_role PERSON 22 35 Anirudh Devgan ROLE 71 93 Chief Executive Officer 0.9889985 works_for PERSON 22 35 Anirudh Devgan ORG 98 104 Cadence 0.9983778 has_role_in_company ROLE 57 65 President ORG 98 104 Cadence 0.9997348 has_role_in_company ROLE 71 93 Chief Executive Officer ORG 98 104 Cadence 0.99845624 has_role ROLE 150 172 Chief Executive Officer PERSON 175 184 Dr. Devgan 0.85268635 has_role_in_company ROLE 150 172 Chief Executive Officer ORG 209 215 Cadence 0.9976404 has_role PERSON 175 184 Dr. Devgan ROLE 196 204 President 0.99899226 works_for PERSON 175 184 Dr. Devgan ORG 209 215 Cadence 0.99876934 has_role_in_company ROLE 196 204 President ORG 209 215 Cadence 0.9997203 has_role PERSON 232 238 Mr. Tan ROLE 268 282 Executive Chair 0.98612714 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finre_work_experience_md| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.7 MB| ## References Manual annotations on CUAD dataset, 10K filings and Wikidata ## Benchmarking ```bash label Recall Precision F1 Support had_role_until 1.000 1.000 1.000 117 has_role 0.998 0.995 0.997 649 has_role_from 1.000 1.000 1.000 401 has_role_in_company 0.993 0.993 0.993 268 other 0.996 0.996 0.996 235 works_for 0.994 1.000 0.997 330 Avg. 0.997 0.997 0.997 2035 Weighted-Avg. 0.997 0.997 0.997 2035 ``` --- layout: model title: Detect PHI in text (enriched-biobert) author: John Snow Labs name: ner_deid_enriched_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect sensitive information in text for de-identification using pretrained NER model. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `DOCTOR`, `PHONE`, `COUNTRY`, `MEDICALRECORD`, `STREET`, `CITY`, `PROFESSION`, `PATIENT`, `IDNUM`, `BIOID`, `HEALTHPLAN`, `HOSPITAL`, `USERNAME`, `LOCATION-OTHER`, `AGE`, `FAX`, `EMAIL`, `DATE`, `STATE`, `ZIP`, `URL`, `ORGANIZATION`, `DEVICE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_en_3.0.0_3.0_1617260810027.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_en_3.0.0_3.0_1617260810027.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_enriched_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_deid_enriched_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.enriched_biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_enriched_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: English asr_Central_kurdish_xlsr TFWav2Vec2ForCTC from Akashpb13 author: John Snow Labs name: asr_Central_kurdish_xlsr date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Central_kurdish_xlsr` is a English model originally trained by Akashpb13. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Central_kurdish_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Central_kurdish_xlsr_en_4.2.0_3.0_1664103765643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Central_kurdish_xlsr_en_4.2.0_3.0_1664103765643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Central_kurdish_xlsr", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Central_kurdish_xlsr", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Central_kurdish_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Sentence Entity Resolver for RxNorm (Action / Treatment) author: John Snow Labs name: sbiobertresolve_rxnorm_action_treatment date: 2022-04-25 tags: [licensed, en, entity_resolution, clinical, rxnorm] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Additionally, this model returns actions and treatments of the drugs in `all_k_aux_labels` column. ## Predicted Entities `RxNorm Codes`, `Action`, `Treatment` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_action_treatment_en_3.5.1_2.4_1650899853599.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_action_treatment_en_3.5.1_2.4_1650899853599.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_action_treatment", "en", "clinical/models")\ .setInputCols(["sbert_embeddings"])\ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver ]) light_model = LightPipeline(pipelineModel) result = light_model.fullAnnotate(["Zita 200 mg", "coumadin 5 mg", "avandia 4 mg"]) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_action_treatment", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val light_model = LightPipeline(rxnorm_pipelineModel) val result = light_model.fullAnnotate(Array("Zita 200 mg", "coumadin 5 mg", "avandia 4 mg")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm_action_treatment").predict("""coumadin 5 mg""") ```

## Results ```bash | | ner_chunk | rxnorm_code | action | treatment | |---:|:--------------|--------------:|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | Zita 200 mg | 104080 | ['Analgesic', 'Antacid', 'Antipyretic', 'Pain Reliever'] | ['Backache', 'Pain', 'Sore Throat', 'Headache', 'Influenza', 'Toothache', 'Heartburn', 'Migraine', 'Muscular Aches And Pains', 'Neuralgia', 'Cold', 'Weakness'] | | 1 | coumadin 5 mg | 855333 | ['Anticoagulant'] | ['Cerebrovascular Accident', 'Pulmonary Embolism', 'Heart Attack', 'AF', 'Embolization'] | | 2 | avandia 4 mg | 261242 | ['Drugs Used In Diabets', 'Hypoglycemic'] | ['Diabetes Mellitus', 'Type 1 Diabetes Mellitus', 'Type 2 Diabetes'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_action_treatment| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|918.7 MB| |Case sensitive:|false| --- layout: model title: English asr_wav2vec2_large_robust_swbd_300h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_large_robust_swbd_300h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_swbd_300h` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_robust_swbd_300h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_swbd_300h_en_4.2.0_3.0_1664038284772.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_swbd_300h_en_4.2.0_3.0_1664038284772.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_robust_swbd_300h", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_robust_swbd_300h", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_robust_swbd_300h| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|757.5 MB| --- layout: model title: English BertForQuestionAnswering model (from maroo93) author: John Snow Labs name: bert_qa_squad2.0 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad2.0` is a English model orginally trained by `maroo93`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad2.0_en_4.0.0_3.0_1654192176164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad2.0_en_4.0.0_3.0_1654192176164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/maroo93/squad2.0 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from vkrishnamoorthy) author: John Snow Labs name: distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `vkrishnamoorthy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773167222.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773167222.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vkrishnamoorthy_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vkrishnamoorthy/distilbert-base-uncased-finetuned-squad --- layout: model title: Named Entity Recognition for Japanese (BERT Base Japanese) author: John Snow Labs name: ner_ud_gsd_bert_base_japanese date: 2021-09-16 tags: [ja, ner, open_sourve, open_source] task: Named Entity Recognition language: ja edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates named entities in a text, that can be used to find features such as names of people, places, and organizations. The model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. This model uses the pretrained BertEmbeddings embeddings "bert_base_ja" as an input, so be sure to use the same embeddings in the pipeline. ## Predicted Entities `ORDINAL`, `PERSON`, `LAW`, `MOVEMENT`, `LOC`, `WORK_OF_ART`, `DATE`, `NORP`, `TITLE_AFFIX`, `QUANTITY`, `FAC`, `TIME`, `MONEY`, `LANGUAGE`, `GPE`, `EVENT`, `ORG`, `PERCENT`, `PRODUCT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ner_ud_gsd_bert_base_japanese_ja_3.2.2_3.0_1631804789491.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ner_ud_gsd_bert_base_japanese_ja_3.2.2_3.0_1631804789491.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from pyspark.ml import Pipeline from sparknlp.annotator import * from sparknlp.base import * from sparknlp.training import * documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") \ .setInputCols(["sentence"]) \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_base_japanese", "ja") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nerTagger = NerDLModel.pretrained("ner_ud_gsd_bert_base_japanese", "ja") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") pipeline = Pipeline().setStages( [ documentAssembler, sentence, word_segmenter, embeddings, nerTagger, ] ) data = spark.createDataFrame([["宮本茂氏は、日本の任天堂のゲームプロデューサーです。"]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.selectExpr("explode(arrays_zip(token.result, ner.result))").show() ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.DocumentAssembler import com.johnsnowlabs.nlp.annotator.{SentenceDetector, WordSegmenterModel} import com.johnsnowlabs.nlp.embeddings.BertEmbeddings import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") .setInputCols("sentence") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_base_japanese", "ja") .setInputCols("sentence", "token") .setOutputCol("embeddings") val nerTagger = NerDLModel.pretrained("ner_ud_gsd_bert_base_japanese", "ja") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentence, word_segmenter, embeddings, nerTagger )) val data = Seq("宮本茂氏は、日本の任天堂のゲームプロデューサーです。").toDF("text") val model = pipeline.fit(data) val result = model.transform(data) result.selectExpr("explode(arrays_zip(token.result, ner.result))").show() ```

## Results ```bash # +-------------------+ # | col| # +-------------------+ # | {宮本, B-PERSON}| # | {茂, I-PERSON}| # | {氏, O}| # | {は, O}| # | {、, O}| # | {日本, B-GPE}| # | {の, O}| # | {任天, B-ORG}| # | {堂, I-ORG}| # | {の, O}| # | {ゲーム, O}| # |{プロデューサー, O}| # | {です, O}| # | {。, O}| # +-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ud_gsd_bert_base_japanese| |Type:|ner| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ja| |Dependencies:|bert_base_ja| ## Data Source The model was trained on the Universal Dependencies, curated by Google. A NER version was created by megagonlabs: https://github.com/megagonlabs/UD_Japanese-GSD Reference: Asahara, M., Kanayama, H., Tanaka, T., Miyao, Y., Uematsu, S., Mori, S., Matsumoto, Y., Omura, M., & Murawaki, Y. (2018). Universal Dependencies Version 2 for Japanese. In LREC-2018. ## Benchmarking ```bash label precision recall f1-score support CARDINAL 0.00 0.00 0.00 0 DATE 0.95 0.96 0.96 206 EVENT 0.84 0.50 0.63 52 FAC 0.75 0.71 0.73 59 GPE 0.79 0.76 0.78 102 LANGUAGE 1.00 1.00 1.00 8 LAW 1.00 0.31 0.47 13 LOC 0.89 0.83 0.86 41 MONEY 1.00 1.00 1.00 20 MOVEMENT 1.00 0.18 0.31 11 NORP 0.85 0.82 0.84 57 O 0.99 0.99 0.99 11785 ORDINAL 0.81 0.94 0.87 32 ORG 0.78 0.65 0.71 179 PERCENT 0.89 1.00 0.94 16 PERSON 0.76 0.84 0.80 127 PRODUCT 0.62 0.68 0.65 50 QUANTITY 0.92 0.94 0.93 172 TIME 0.97 0.88 0.92 32 TITLE_AFFIX 0.89 0.71 0.79 24 WORK_OF_ART 0.66 0.73 0.69 48 accuracy - - 0.97 13034 macro-avg 0.83 0.73 0.75 13034 weighted-avg 0.97 0.97 0.97 13034 ``` --- layout: model title: English BertForTokenClassification Small Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_modified_PubmedBert_small date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4-modified-PubmedBert_small` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_modified_PubmedBert_small_en_4.0.0_3.0_1657108180187.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_modified_PubmedBert_small_en_4.0.0_3.0_1657108180187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_modified_PubmedBert_small| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4-modified-PubmedBert_small --- layout: model title: Pipeline to Detect Disease Mentions (MedicalBertForTokenClassification) (BERT) author: John Snow Labs name: bert_token_classifier_disease_mentions_tweet_pipeline date: 2023-03-20 tags: [es, clinical, licensed, public_health, ner, token_classification, disease, tweet] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_disease_mentions_tweet](https://nlp.johnsnowlabs.com/2022/07/28/bert_token_classifier_disease_mentions_tweet_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_disease_mentions_tweet_pipeline_es_4.3.0_3.2_1679299531828.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_disease_mentions_tweet_pipeline_es_4.3.0_3.2_1679299531828.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_disease_mentions_tweet_pipeline", "es", "clinical/models") text = '''El diagnóstico fueron varios. Principal: Neumonía en el pulmón derecho. Sinusitis de caballo, Faringitis aguda e infección de orina, también elevada. Gripe No. Estuvo hablando conmigo, sin exagerar, mas de media hora, dándome ánimo y fuerza y que sabe, porque ha visto.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_disease_mentions_tweet_pipeline", "es", "clinical/models") val text = "El diagnóstico fueron varios. Principal: Neumonía en el pulmón derecho. Sinusitis de caballo, Faringitis aguda e infección de orina, también elevada. Gripe No. Estuvo hablando conmigo, sin exagerar, mas de media hora, dándome ánimo y fuerza y que sabe, porque ha visto." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:------------|-------------:| | 0 | Neumonía en el pulmón | 41 | 61 | ENFERMEDAD | 0.999969 | | 1 | Sinusitis | 72 | 80 | ENFERMEDAD | 0.999977 | | 2 | Faringitis aguda | 94 | 109 | ENFERMEDAD | 0.999969 | | 3 | infección de orina | 113 | 130 | ENFERMEDAD | 0.999969 | | 4 | Gripe | 150 | 154 | ENFERMEDAD | 0.999983 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_disease_mentions_tweet_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|462.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_nl16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-nl16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl16_en_4.3.0_3.0_1675113699074.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl16_en_4.3.0_3.0_1675113699074.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_nl16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_nl16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_nl16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|602.0 MB| ## References - https://huggingface.co/google/t5-efficient-base-nl16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English asr_wav2vec2_base_100h_by_vuiseng9 TFWav2Vec2ForCTC from vuiseng9 author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_by_vuiseng9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_by_vuiseng9` is a English model originally trained by vuiseng9. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_by_vuiseng9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_by_vuiseng9_en_4.2.0_3.0_1664022865816.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_by_vuiseng9_en_4.2.0_3.0_1664022865816.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_by_vuiseng9', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_by_vuiseng9", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_by_vuiseng9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English image_classifier_vit_taco_or_what ViTForImageClassification from osanseviero author: John Snow Labs name: image_classifier_vit_taco_or_what date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_taco_or_what` is a English model originally trained by osanseviero. ## Predicted Entities `burrito`, `taco`, `quesadilla`, `fajitas`, `kebab` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_taco_or_what_en_4.1.0_3.0_1660169560946.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_taco_or_what_en_4.1.0_3.0_1660169560946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_taco_or_what", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_taco_or_what", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_taco_or_what| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Sentence Embeddings - sbert medium (tuned) author: John Snow Labs name: sbert_jsl_medium_umls_uncased date: 2021-05-14 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_umls_uncased_en_3.0.3_2.4_1621017148548.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_umls_uncased_en_3.0.3_2.4_1621017148548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_umls_uncased","en","clinical/models")\ .setInputCols(["sentence"])\ .setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_umls_uncased","en","clinical/models") .setInputCols("sentence") .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_medium_umls_uncased").predict("""Put your text here.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ke_base","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ke_base","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_ke_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|569.3 MB| ## References - https://huggingface.co/KETI-AIR/ke-t5-base - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://github.com/google-research/text-to-text-transfer-transformer#released-model-checkpoints - https://github.com/AIRC-KETI/ke-t5 - https://aclanthology.org/2021.findings-emnlp.33/ - https://jmlr.org/papers/volume21/20-074/20-074.pdf - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://aclanthology.org/2021.acl-long.330.pdf - https://dl.acm.org/doi/pdf/10.1145/3442188.3445922 - https://www.tensorflow.org/datasets/catalog/c4 - https://jmlr.org/papers/volume21/20-074/20-074.pdf - https://jmlr.org/papers/volume21/20-074/20-074.pdf - https://jmlr.org/papers/volume21/20-074/20-074.pdf - https://mlco2.github.io/impact#compute - https://arxiv.org/abs/1910.09700 - https://colab.research.google.com/github/google-research/text-to-text-transfer-transformer/blob/main/notebooks/t5-trivia.ipynb --- layout: model title: Translate Portuguese-based languages to English Pipeline author: John Snow Labs name: translate_cpp_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, cpp, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `cpp` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_cpp_en_xx_2.7.0_2.4_1609687142938.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_cpp_en_xx_2.7.0_2.4_1609687142938.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_cpp_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_cpp_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.cpp.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_cpp_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Cased model (from mcurmei) author: John Snow Labs name: distilbert_qa_single_label_n_max date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `single_label_N_max` is a English model originally trained by `mcurmei`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_single_label_n_max_en_4.3.0_3.0_1672775498618.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_single_label_n_max_en_4.3.0_3.0_1672775498618.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_single_label_n_max","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_single_label_n_max","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_single_label_n_max| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mcurmei/single_label_N_max --- layout: model title: Japanese Electra Embeddings (from izumi-lab) author: John Snow Labs name: electra_embeddings_electra_small_japanese_fin_generator date: 2022-05-17 tags: [ja, open_source, electra, embeddings] task: Embeddings language: ja edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-small-japanese-fin-generator` is a Japanese model orginally trained by `izumi-lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_small_japanese_fin_generator_ja_3.4.4_3.0_1652786680826.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_small_japanese_fin_generator_ja_3.4.4_3.0_1652786680826.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_small_japanese_fin_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ja| |Size:|52.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/izumi-lab/electra-small-japanese-fin-generator - https://github.com/google-research/electra - https://github.com/retarfi/language-pretraining/tree/v1.0 - https://github.com/google-research/electra - https://arxiv.org/abs/2003.10555 - https://creativecommons.org/licenses/by-sa/4.0/ --- layout: model title: English ElectraForQuestionAnswering model (from ptran74) Version-5 author: John Snow Labs name: electra_qa_DSPFirst_Finetuning_5 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `DSPFirst-Finetuning-5` is a English model originally trained by `ptran74`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_5_en_4.0.0_3.0_1655919805104.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_5_en_4.0.0_3.0_1655919805104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_5","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_5","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.electra.finetuning_5").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_DSPFirst_Finetuning_5| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ptran74/DSPFirst-Finetuning-5 - https://github.gatech.edu/pages/VIP-ITS/textbook_SQuAD_explore/explore/textbookv1.0/textbook/ - https://github.com/patil-suraj/question_generation --- layout: model title: Wolof XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof date: 2022-08-01 tags: [wo, open_source, xlm_roberta, ner] task: Named Entity Recognition language: wo edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-wolof-finetuned-ner-wolof` is a Wolof model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof_wo_4.1.0_3.0_1659356140998.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof_wo_4.1.0_3.0_1659356140998.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof","wo") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof","wo") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_wolof_finetuned_ner_wolof| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|wo| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-wolof-finetuned-ner-wolof - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Legal Guarantee Clause Binary Classifier author: John Snow Labs name: legclf_guarantee_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `guarantee` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `guarantee` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_guarantee_clause_en_1.0.0_3.2_1660123571558.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_guarantee_clause_en_1.0.0_3.2_1660123571558.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_guarantee_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[guarantee]| |[other]| |[other]| |[guarantee]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_guarantee_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support guarantee 0.88 0.80 0.83 88 other 0.91 0.95 0.93 192 accuracy - - 0.90 280 macro-avg 0.89 0.87 0.88 280 weighted-avg 0.90 0.90 0.90 280 ``` --- layout: model title: English ElectraForQuestionAnswering Small model (from mrm8488) author: John Snow Labs name: electra_qa_small_finetuned_squadv1 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-small-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_small_finetuned_squadv1_en_4.0.0_3.0_1655921278345.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_small_finetuned_squadv1_en_4.0.0_3.0_1655921278345.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_small_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_small_finetuned_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.small").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_small_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|51.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/electra-small-finetuned-squadv1 - https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/ --- layout: model title: Legal Negative covenants Clause Binary Classifier author: John Snow Labs name: legclf_negative_covenants_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `negative-covenants` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `negative-covenants` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_negative_covenants_clause_en_1.0.0_3.2_1660122676325.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_negative_covenants_clause_en_1.0.0_3.2_1660122676325.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_negative_covenants_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[negative-covenants]| |[other]| |[other]| |[negative-covenants]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_negative_covenants_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support negative-covenants 1.00 0.92 0.96 51 other 0.97 1.00 0.98 130 accuracy - - 0.98 181 macro-avg 0.99 0.96 0.97 181 weighted-avg 0.98 0.98 0.98 181 ``` --- layout: model title: Detect Organism in Medical Text author: John Snow Labs name: bert_token_classifier_ner_species date: 2022-07-25 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Species-800 is a corpus for species entities, which is based on manually annotated abstracts. It comprises 800 PubMed abstracts that contain identified organism mentions. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_species_en_4.0.0_3.0_1658758056681.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_species_en_4.0.0_3.0_1658758056681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------+-------+ |ner_chunk |label | +-----------------------+-------+ |6C (T) |SPECIES| |Betaproteobacteria |SPECIES| |Thiomonas intermedia |SPECIES| |DSM 18155 (T) |SPECIES| |Thiomonas perometabolis|SPECIES| |DSM 18570 (T) |SPECIES| +-----------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_species| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://species.jensenlab.org/](https://species.jensenlab.org/) ## Benchmarking ```bash label precision recall f1-score support B-SPECIES 0.6073 0.9374 0.7371 767 I-SPECIES 0.7418 0.8648 0.7986 1043 micro-avg 0.6754 0.8956 0.7701 1810 macro-avg 0.6745 0.9011 0.7678 1810 weighted-avg 0.6848 0.8956 0.7725 1810 ``` --- layout: model title: Sentiment Analysis pipeline for English (analyze_sentimentdl_glove_imdb) author: John Snow Labs name: analyze_sentimentdl_glove_imdb date: 2021-03-24 tags: [open_source, english, analyze_sentimentdl_glove_imdb, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The analyze_sentimentdl_glove_imdb is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_glove_imdb_en_3.0.0_3.0_1616544505213.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_glove_imdb_en_3.0.0_3.0_1616544505213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('analyze_sentimentdl_glove_imdb', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("analyze_sentimentdl_glove_imdb", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.sentiment.glove').predict(text) result_df ```

## Results ```bash | | document | sentence | tokens | word_embeddings | sentence_embeddings | sentiment | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:-----------------------------|:------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[0.2668800055980682,.,...]] | [[0.0771183446049690,.,...]] | ['neg'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|analyze_sentimentdl_glove_imdb| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: Extract Pharmacological Entities From Spanish Medical Texts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_pharmacology date: 2022-08-11 tags: [es, clinical, licensed, token_classification, bert, ner, pharmacology] task: Named Entity Recognition language: es edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Named Entity Recognition model is intended for detecting pharmacological entities from Spanish medical texts and trained using the BertForTokenClassification method from the transformers library and [BERT based](https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased) embeddings. The model detects PROTEINAS and NORMALIZABLES. ## Predicted Entities `PROTEINAS`, `NORMALIZABLES` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_pharmacology_es_4.0.2_3.0_1660236427687.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_pharmacology_es_4.0.2_3.0_1660236427687.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_pharmacology", "es", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("label")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["sentence","token","label"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) data = spark.createDataFrame([["""Se realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa)."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_pharmacology", "es", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("label") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("sentence","token","label")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter)) val data = Seq(Array("Se realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa).")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.classify.bert_token.pharmacology").predict("""Se realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa).""") ```

## Results ```bash +---------------+-------------+ |chunk |ner_label | +---------------+-------------+ |creatinkinasa |PROTEINAS | |LDH |PROTEINAS | |urea |NORMALIZABLES| |CA 19.9 |PROTEINAS | |vimentina |PROTEINAS | |S-100 |PROTEINAS | |HMB-45 |PROTEINAS | |actina |PROTEINAS | |Cisplatino |NORMALIZABLES| |Interleukina II|PROTEINAS | |Dacarbacina |NORMALIZABLES| |Interferon alfa|PROTEINAS | +---------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_pharmacology| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## Benchmarking ```bash label precision recall f1-score support B-NORMALIZABLES 0.9458 0.9694 0.9575 3076 I-NORMALIZABLES 0.8788 0.8969 0.8878 291 B-PROTEINAS 0.9164 0.9369 0.9265 2234 I-PROTEINAS 0.8825 0.7634 0.8186 748 micro-avg 0.9257 0.9304 0.9280 6349 macro-avg 0.9059 0.8917 0.8976 6349 weighted-avg 0.9249 0.9304 0.9270 6349 ``` --- layout: model title: Translate Manx to English Pipeline author: John Snow Labs name: translate_gv_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, gv, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `gv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_gv_en_xx_2.7.0_2.4_1609686733139.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_gv_en_xx_2.7.0_2.4_1609686733139.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gv.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gv_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Death Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_death_bert date: 2023-03-05 tags: [en, legal, classification, clauses, death, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Death` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Death`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_death_bert_en_1.0.0_3.0_1678049968182.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_death_bert_en_1.0.0_3.0_1678049968182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_death_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Death]| |[Other]| |[Other]| |[Death]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_death_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Death 0.86 1.00 0.93 31 Other 1.00 0.90 0.95 49 accuracy - - 0.94 80 macro-avg 0.93 0.95 0.94 80 weighted-avg 0.95 0.94 0.94 80 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from KES) author: John Snow Labs name: t5_kes date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `T5-KES` is a English model originally trained by `KES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_kes_en_4.3.0_3.0_1675099343508.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_kes_en_4.3.0_3.0_1675099343508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_kes","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_kes","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_kes| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|912.8 MB| ## References - https://huggingface.co/KES/T5-KES - https://arxiv.org/abs/1702.04066 - https://github.com/EricFillion/happy-transformer - https://pypi.org/project/Caribe/ --- layout: model title: Legal Sanctions Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_sanctions_bert date: 2023-03-05 tags: [en, legal, classification, clauses, sanctions, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Sanctions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Sanctions`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sanctions_bert_en_1.0.0_3.0_1678050581574.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sanctions_bert_en_1.0.0_3.0_1678050581574.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_sanctions_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Sanctions]| |[Other]| |[Other]| |[Sanctions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sanctions_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 1.00 0.95 0.97 19 Sanctions 0.92 1.00 0.96 11 accuracy - - 0.97 30 macro-avg 0.96 0.97 0.96 30 weighted-avg 0.97 0.97 0.97 30 ``` --- layout: model title: Word2Vec Embeddings in Uzbek (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, uz, open_source] task: Embeddings language: uz edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_uz_3.4.1_3.0_1647465690254.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_uz_3.4.1_3.0_1647465690254.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_sl", "sl") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("John Snow je poleg tega, da je severni kralj, angleški zdravnik in vodilni v razvoju anestezije in zdravstvene higiene.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_sl", "sl") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("John Snow je poleg tega, da je severni kralj, angleški zdravnik in vodilni v razvoju anestezije in zdravstvene higiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""John Snow je poleg tega, da je severni kralj, angleški zdravnik in vodilni v razvoju anestezije in zdravstvene higiene."""] stopword_df = nlu.load('sl.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=3, result='John', metadata={'sentence': '0'}), Row(annotatorType='token', begin=5, end=8, result='Snow', metadata={'sentence': '0'}), Row(annotatorType='token', begin=23, end=23, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=31, end=37, result='severni', metadata={'sentence': '0'}), Row(annotatorType='token', begin=39, end=43, result='kralj', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_sl| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|sl| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Financial Finetuned FLAN-T5 Text Generation (FIQA dataset) author: John Snow Labs name: fingen_flant5_finetuned_fiqa date: 2023-05-29 tags: [en, finance, generation, licensed, flant5, fiqa, tensorflow] task: Text Generation language: en edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: FinanceTextGenerator article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `fingen_flant5_finetuned_fiqa` model is the Text Generation model that has been fine-tuned on FLAN-T5 using FIQA dataset. FLAN-T5 is a state-of-the-art language model developed by Google AI that utilizes the T5 architecture for text-generation tasks. References: ```bibtex @article{flant5_paper, title={Scaling instruction-finetuned language models}, author={Chung, Hyung Won and Hou, Le and Longpre, Shayne and Zoph, Barret and Tay, Yi and Fedus, William and Li, Eric and Wang, Xuezhi and Dehghani, Mostafa and Brahma, Siddhartha and others}, journal={arXiv preprint arXiv:2210.11416}, year={2022} } @article{t5_paper, title={Exploring the limits of transfer learning with a unified text-to-text transformer}, author={Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J}, journal={The Journal of Machine Learning Research}, volume={21}, number={1}, pages={5485--5551}, year={2020}, publisher={JMLRORG} } ``` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/fingen_flant5_finetuned_fiqa_en_1.0.0_3.0_1685363340017.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/fingen_flant5_finetuned_fiqa_en_1.0.0_3.0_1685363340017.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") flant5 = finance.TextGenerator.pretrained("fingen_flant5_finetuned_fiqa", "en", "finance/models")\ .setInputCols(["document"])\ .setOutputCol("generated")\ .setMaxNewTokens(256)\ .setStopAtEos(True)\ .setDoSample(True)\ .setTopK(3) pipeline = nlp.Pipeline(stages=[document_assembler, flant5]) data = spark.createDataFrame([ [1, "How to have a small capital investment in US if I am out of the country?"]]).toDF('id', 'text') results = pipeline.fit(data).transform(data) results.select("generated.result").show(truncate=False) ```

## Results ```bash +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |result | +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[I would suggest a local broker. They have diversified funds that are diversified and have the same fees as the US market. They also offer diversified portfolios that have the lowest risk.]| +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|fingen_flant5_finetuned_fiqa| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.6 GB| ## References The dataset is available [here](https://huggingface.co/datasets/BeIR/fiqa) --- layout: model title: Extract Intent Type from Customer Service Chat Messages author: John Snow Labs name: finclf_customer_service_intent_type date: 2023-02-03 tags: [en, licensed, intent, finance, customer, tensorflow] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: FinanceClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Text Classification model that can help you classify a chat message from customer service according to intent type. ## Predicted Entities `cancel_order`, `change_order`, `change_setup_shipping_address`, `check_cancellation_fee`, `check_payment_methods`, `check_refund_policy`, `complaint`, `contact_customer_service`, `contact_human_agent`, `create_edit_switch_account`, `delete_account`, `delivery_options`, `delivery_period`, `get_check_invoice`, `get_refund`, `newsletter_subscription`, `payment_issue`, `place_order`, `recover_password`, `registration_problems`, `review`, `track_order`, `track_refund`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_customer_service_intent_type_en_1.0.0_3.0_1675427852317.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_customer_service_intent_type_en_1.0.0_3.0_1675427852317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") embeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = finance.ClassifierDLModel.pretrained("finclf_customer_service_intent_type", "en", "finance/models")\ .setInputCols("sentence_embeddings") \ .setOutputCol("class") pipeline = nlp.Pipeline().setStages( [ document_assembler, embeddings, docClassifier ] ) empty_data = spark.createDataFrame([[""]]).toDF("text") model = pipeline.fit(empty_data) light_model = nlp.LightPipeline(model) result = light_model.annotate("""I have a problem with the deletion of my Premium account.""") result["class"] ```

## Results ```bash ['delete_account'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_customer_service_intent_type| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References https://github.com/bitext/customer-support-intent-detection-evaluation-dataset ## Benchmarking ```bash label precision recall f1-score support cancel_order 0.88 1.00 0.94 30 change_order 1.00 0.90 0.95 30 change_setup_shipping_address 0.97 0.97 0.97 36 check_cancellation_fee 0.97 0.97 0.97 30 check_payment_methods 0.97 0.93 0.95 30 check_refund_policy 0.97 0.97 0.97 30 complaint 0.93 0.93 0.93 30 contact_customer_service 1.00 1.00 1.00 30 contact_human_agent 0.97 0.97 0.97 30 create_edit_switch_account 0.90 0.97 0.93 36 delete_account 0.96 0.87 0.91 30 delivery_options 0.91 1.00 0.95 30 delivery_period 1.00 0.97 0.98 30 get_check_invoice 0.92 0.97 0.95 36 get_refund 1.00 0.87 0.93 30 newsletter_subscription 1.00 0.93 0.97 30 other 1.00 0.92 0.96 38 payment_issue 0.97 1.00 0.98 30 place_order 0.97 0.93 0.95 30 recover_password 0.97 1.00 0.98 30 registration_problems 1.00 0.97 0.98 30 review 0.94 1.00 0.97 30 track_order 0.93 0.93 0.93 30 track_refund 0.91 1.00 0.95 30 accuracy - - 0.96 746 macro-avg 0.96 0.96 0.96 746 weighted-avg 0.96 0.96 0.96 746 ``` --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_200000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-200000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_200000_cased_generator_de_3.4.4_3.0_1652786339074.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_200000_cased_generator_de_3.4.4_3.0_1652786339074.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_200000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-200000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: Arabic Named Entity Recognition (from abdusahmbzuai) author: John Snow Labs name: bert_ner_arabert_ner date: 2022-05-04 tags: [bert, ner, token_classification, ar, open_source] task: Named Entity Recognition language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `arabert-ner` is a Arabic model orginally trained by `abdusahmbzuai`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_arabert_ner_ar_3.4.2_3.0_1651630356143.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_arabert_ner_ar_3.4.2_3.0_1651630356143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_arabert_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|505.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/abdusahmbzuai/arabert-ner --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_radhakri119 TFWav2Vec2ForCTC from radhakri119 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_radhakri119 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_radhakri119` is a English model originally trained by radhakri119. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_radhakri119_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_radhakri119_en_4.2.0_3.0_1664101755978.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_radhakri119_en_4.2.0_3.0_1664101755978.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_radhakri119", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_radhakri119", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_radhakri119| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Central Bikol author: John Snow Labs name: opus_mt_en_bcl date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bcl, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bcl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bcl_xx_2.7.0_2.4_1609170612440.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bcl_xx_2.7.0_2.4_1609170612440.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bcl| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_orkg_what_5e_05 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-orkg-what-5e-05` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_what_5e_05_en_4.3.0_3.0_1672766856335.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_what_5e_05_en_4.3.0_3.0_1672766856335.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_what_5e_05","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_what_5e_05","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_orkg_what_5e_05| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Moussab/distilbert-base-cased-distilled-squad-orkg-what-5e-05 --- layout: model title: Javanese BertForMaskedLM Small Cased model (from w11wo) author: John Snow Labs name: bert_embeddings_javanese_small_imdb date: 2022-12-02 tags: [jv, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: jv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `javanese-bert-small-imdb` is a Javanese model originally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_small_imdb_jv_4.2.4_3.0_1670022513681.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_small_imdb_jv_4.2.4_3.0_1670022513681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_javanese_small_imdb","jv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_javanese_small_imdb","jv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_javanese_small_imdb| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|410.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-bert-small-imdb - https://arxiv.org/abs/1810.04805 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: Persian Named Entity Recognition (from HooshvareLab) author: John Snow Labs name: bert_ner_bert_base_parsbert_peymaner_uncased date: 2022-05-09 tags: [bert, ner, token_classification, fa, open_source] task: Named Entity Recognition language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-parsbert-peymaner-uncased` is a Persian model orginally trained by `HooshvareLab`. ## Predicted Entities `LOC`, `PER`, `TIM`, `MON`, `DAT`, `PCT`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_parsbert_peymaner_uncased_fa_3.4.2_3.0_1652099544405.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_parsbert_peymaner_uncased_fa_3.4.2_3.0_1652099544405.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_parsbert_peymaner_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|607.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/HooshvareLab/bert-base-parsbert-peymaner-uncased - https://arxiv.org/abs/2005.12515 - http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/ - https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://arxiv.org/abs/2005.12515 - https://tensorflow.org/tfrc - https://hooshvare.com - https://www.linkedin.com/in/m3hrdadfi/ - https://twitter.com/m3hrdadfi - https://github.com/m3hrdadfi - https://www.linkedin.com/in/mohammad-gharachorloo/ - https://twitter.com/MGharachorloo - https://github.com/baarsaam - https://www.linkedin.com/in/marziehphi/ - https://twitter.com/marziehphi - https://github.com/marziehphi - https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/ - https://twitter.com/mmanthouri - https://github.com/mmanthouri - https://hooshvare.com/ - https://www.linkedin.com/company/hooshvare - https://twitter.com/hooshvare - https://github.com/hooshvare - https://www.instagram.com/hooshvare/ - https://www.linkedin.com/in/sara-tabrizi-64548b79/ - https://www.behance.net/saratabrizi - https://www.instagram.com/sara_b_tabrizi/ --- layout: model title: Explain Document Pipeline for Norwegian (Bokmal) author: John Snow Labs name: explain_document_sm date: 2021-03-22 tags: [open_source, norwegian_bokmal, explain_document_sm, pipeline, "no"] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: "no" edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_sm_no_3.0.0_3.0_1616427435939.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_sm_no_3.0.0_3.0_1616427435939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_sm', lang = 'no') annotations = pipeline.fullAnnotate(""Hei fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_sm", lang = "no") val result = pipeline.fullAnnotate("Hei fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei fra John Snow Labs! ""] result_df = nlu.load('no.explain').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:--------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hei fra John Snow Labs! '] | ['Hei fra John Snow Labs!'] | ['Hei', 'fra', 'John', 'Snow', 'Labs!'] | ['Hei', 'fra', 'John', 'Snow', 'Labs!'] | ['PROPN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[-0.394499987363815,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|no| --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_ff9000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-ff9000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff9000_en_4.3.0_3.0_1675123598242.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff9000_en_4.3.0_3.0_1675123598242.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_ff9000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_ff9000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_ff9000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|110.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-ff9000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English RobertaForMaskedLM Base Cased model author: John Snow Labs name: roberta_embeddings_distil_base date: 2022-12-12 tags: [en, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-base` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distil_base_en_4.2.4_3.0_1670858593481.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distil_base_en_4.2.4_3.0_1670858593481.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_distil_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_distil_base","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_distil_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|308.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/distilroberta-base - https://arxiv.org/abs/1910.01108 - https://aclanthology.org/2021.acl-long.330.pdf - https://dl.acm.org/doi/pdf/10.1145/3442188.3445922 - https://skylion007.github.io/OpenWebTextCorpus/ - https://mlco2.github.io/impact#compute - https://arxiv.org/abs/1910.09700 --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-few-shot-k-1024-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4_en_4.3.0_3.0_1674221411536.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4_en_4.3.0_3.0_1674221411536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_few_shot_k_1024_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-large-few-shot-k-1024-finetuned-squad-seed-4 --- layout: model title: English BertForQuestionAnswering model (from MrAnderson) author: John Snow Labs name: bert_qa_bert_base_1024_full_trivia_copied_embeddings date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-1024-full-trivia-copied-embeddings` is a English model orginally trained by `MrAnderson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_1024_full_trivia_copied_embeddings_en_4.0.0_3.0_1654179607765.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_1024_full_trivia_copied_embeddings_en_4.0.0_3.0_1654179607765.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_1024_full_trivia_copied_embeddings","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_1024_full_trivia_copied_embeddings","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.trivia.bert.base_1024d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_1024_full_trivia_copied_embeddings| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|409.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MrAnderson/bert-base-1024-full-trivia-copied-embeddings --- layout: model title: Legal Transfer Clause Binary Classifier author: John Snow Labs name: legclf_transfer_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `transfer` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `transfer` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_transfer_clause_en_1.0.0_3.2_1660124097088.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_transfer_clause_en_1.0.0_3.2_1660124097088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_transfer_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[transfer]| |[other]| |[other]| |[transfer]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_transfer_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.91 0.94 0.93 66 transfer 0.90 0.86 0.88 43 accuracy - - 0.91 109 macro-avg 0.91 0.90 0.90 109 weighted-avg 0.91 0.91 0.91 109 ``` --- layout: model title: Fast Neural Machine Translation Model from Tumbuka to English author: John Snow Labs name: opus_mt_tum_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tum, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tum` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tum_en_xx_2.7.0_2.4_1609168643029.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tum_en_xx_2.7.0_2.4_1609168643029.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tum_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_bilal_20epoch TFWav2Vec2ForCTC from Roshana author: John Snow Labs name: pipeline_asr_wav2vec2_bilal_20epoch date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_bilal_20epoch` is a English model originally trained by Roshana. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_bilal_20epoch_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_bilal_20epoch_en_4.2.0_3.0_1664119706366.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_bilal_20epoch_en_4.2.0_3.0_1664119706366.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_bilal_20epoch', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_bilal_20epoch", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_bilal_20epoch| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Luba-Lulua author: John Snow Labs name: opus_mt_en_lua date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, lua, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `lua` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_lua_xx_2.7.0_2.4_1609164365414.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_lua_xx_2.7.0_2.4_1609164365414.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_lua| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Clinical Entities (jsl_ner_wip_clinical) author: John Snow Labs name: jsl_ner_wip_clinical date: 2021-01-18 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.7.0 spark_version: 2.4 tags: [ner, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `Injury_or_Poisoning`, `Direction`, `Test`, `Admission_Discharge`, `Death_Entity`, `Relationship_Status`, `Duration`, `Respiration`, `Hyperlipidemia`, `Birth_Entity`, `Age`, `Labour_Delivery`, `Family_History_Header`, `BMI`, `Temperature`, `Alcohol`, `Kidney_Disease`, `Oncological`, `Medical_History_Header`, `Cerebrovascular_Disease`, `Oxygen_Therapy`, `O2_Saturation`, `Psychological_Condition`, `Heart_Disease`, `Employment`, `Obesity`, `Disease_Syndrome_Disorder`, `Pregnancy`, `ImagingFindings`, `Procedure`, `Medical_Device`, `Race_Ethnicity`, `Section_Header`, `Symptom`, `Treatment`, `Substance`, `Route`, `Drug_Ingredient`, `Blood_Pressure`, `Diet`, `I-Age`, `External_body_part_or_region`, `LDL`, `VS_Finding`, `Allergen`, `EKG_Findings`, `Imaging_Technique`, `I-Diet`, `Triglycerides`, `RelativeTime`, `Gender`, `Pulse`, `Social_History_Header`, `Substance_Quantity`, `Diabetes`, `Modifier`, `Internal_organ_or_component`, `Clinical_Dept`, `Form`, `Drug_BrandName`, `Strength`, `Fetus_NewBorn`, `RelativeDate`, `Height`, `Test_Result`, `Sexually_Active_or_Sexual_Orientation`, `Frequency`, `Time`, `Weight`, `Vaccine`, `Vital_Signs_Header`, `Communicable_Disease`, `Dosage`, `Overweight`, `Hypertension`, `HDL`, `Total_Cholesterol`, `Smoking`, `Date`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_clinical_en_2.6.5_2.4_1609505628141.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_clinical_en_2.6.5_2.4_1609505628141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings_clinical = WordEmbeddingsModel.pretrained('embeddings_clinical', 'en', 'clinical/models') \ .setInputCols(['sentence', 'token']) \ .setOutputCol('embeddings') clinical_ner = NerDLModel.pretrained("jsl_ner_wip_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature."]], ["text"])) ``` ```scala ... val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols("sentence", "token") .setOutputCol("embeddings") val ner = NerDLModel.pretrained("jsl_ner_wip_clinical", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

{:.h2_title} ## Results The output is a dataframe with a sentence per row and a ``"ner"`` column containing all of the entity labels in the sentence, entity character indices, and other metadata. To get only the tokens and entity labels, without the metadata, select ``"token.result"`` and ``"ner.result"`` from your output dataframe or add the ``"Finisher"`` to the end of your pipeline. ```bash +-----------------------------------------+----------------------------+ |chunk |ner_label | +-----------------------------------------+----------------------------+ |21-day-old |Age | |Caucasian |Race_Ethnicity | |male |Gender | |for 2 days |Duration | |congestion |Symptom | |mom |Gender | |yellow |Modifier | |discharge |Symptom | |nares |External_body_part_or_region| |she |Gender | |mild |Modifier | |problems with his breathing while feeding|Symptom | |perioral cyanosis |Symptom | |retractions |Symptom | |One day ago |RelativeDate | |mom |Gender | |Tylenol |Drug_BrandName | |Baby |Age | |decreased p.o. intake |Symptom | |His |Gender | +-----------------------------------------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_clinical| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence,token, embeddings]| |Output Labels:|[ner]| |Language:|[en]| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ {:.h2_title} ## Benchmarking ```bash entity tp fp fn total precision recall f1 VS_Finding 235.0 46.0 43.0 278.0 0.8363 0.8453 0.8408 Direction 3972.0 465.0 458.0 4430.0 0.8952 0.8966 0.8959 Respiration 82.0 4.0 4.0 86.0 0.9535 0.9535 0.9535 Cerebrovascular_D... 93.0 20.0 24.0 117.0 0.823 0.7949 0.8087 Family_History_He... 88.0 6.0 3.0 91.0 0.9362 0.967 0.9514 Heart_Disease 447.0 82.0 119.0 566.0 0.845 0.7898 0.8164 RelativeTime 158.0 80.0 59.0 217.0 0.6639 0.7281 0.6945 Strength 624.0 58.0 53.0 677.0 0.915 0.9217 0.9183 Smoking 121.0 11.0 4.0 125.0 0.9167 0.968 0.9416 Medical_Device 3716.0 491.0 466.0 4182.0 0.8833 0.8886 0.8859 Pulse 136.0 22.0 14.0 150.0 0.8608 0.9067 0.8831 Psychological_Con... 135.0 9.0 29.0 164.0 0.9375 0.8232 0.8766 Overweight 2.0 1.0 0.0 2.0 0.6667 1.0 0.8 Triglycerides 3.0 0.0 2.0 5.0 1.0 0.6 0.75 Obesity 42.0 5.0 6.0 48.0 0.8936 0.875 0.8842 Admission_Discharge 318.0 24.0 11.0 329.0 0.9298 0.9666 0.9478 HDL 3.0 0.0 0.0 3.0 1.0 1.0 1.0 Diabetes 110.0 14.0 8.0 118.0 0.8871 0.9322 0.9091 Section_Header 3740.0 148.0 157.0 3897.0 0.9619 0.9597 0.9608 Age 627.0 75.0 48.0 675.0 0.8932 0.9289 0.9107 O2_Saturation 34.0 14.0 17.0 51.0 0.7083 0.6667 0.6869 Kidney_Disease 96.0 12.0 34.0 130.0 0.8889 0.7385 0.8067 Test 2504.0 545.0 498.0 3002.0 0.8213 0.8341 0.8276 Communicable_Disease 21.0 10.0 6.0 27.0 0.6774 0.7778 0.7241 Hypertension 162.0 5.0 10.0 172.0 0.9701 0.9419 0.9558 External_body_par... 2626.0 356.0 413.0 3039.0 0.8806 0.8641 0.8723 Oxygen_Therapy 81.0 15.0 14.0 95.0 0.8438 0.8526 0.8482 Modifier 2341.0 404.0 539.0 2880.0 0.8528 0.8128 0.8324 Test_Result 1007.0 214.0 255.0 1262.0 0.8247 0.7979 0.8111 BMI 9.0 1.0 0.0 9.0 0.9 1.0 0.9474 Labour_Delivery 57.0 23.0 33.0 90.0 0.7125 0.6333 0.6706 Employment 271.0 59.0 55.0 326.0 0.8212 0.8313 0.8262 Fetus_NewBorn 66.0 33.0 51.0 117.0 0.6667 0.5641 0.6111 Clinical_Dept 923.0 110.0 83.0 1006.0 0.8935 0.9175 0.9053 Time 29.0 13.0 16.0 45.0 0.6905 0.6444 0.6667 Procedure 3185.0 462.0 501.0 3686.0 0.8733 0.8641 0.8687 Diet 36.0 20.0 45.0 81.0 0.6429 0.4444 0.5255 Oncological 459.0 61.0 55.0 514.0 0.8827 0.893 0.8878 LDL 3.0 0.0 3.0 6.0 1.0 0.5 0.6667 Symptom 7104.0 1302.0 1200.0 8304.0 0.8451 0.8555 0.8503 Temperature 116.0 6.0 8.0 124.0 0.9508 0.9355 0.9431 Vital_Signs_Header 215.0 29.0 24.0 239.0 0.8811 0.8996 0.8903 Relationship_Status 49.0 2.0 1.0 50.0 0.9608 0.98 0.9703 Total_Cholesterol 11.0 4.0 5.0 16.0 0.7333 0.6875 0.7097 Blood_Pressure 158.0 18.0 22.0 180.0 0.8977 0.8778 0.8876 Injury_or_Poisoning 579.0 130.0 127.0 706.0 0.8166 0.8201 0.8184 Drug_Ingredient 1716.0 153.0 132.0 1848.0 0.9181 0.9286 0.9233 Treatment 136.0 36.0 60.0 196.0 0.7907 0.6939 0.7391 Pregnancy 123.0 36.0 51.0 174.0 0.7736 0.7069 0.7387 Vaccine 13.0 2.0 6.0 19.0 0.8667 0.6842 0.7647 Disease_Syndrome_... 2981.0 559.0 446.0 3427.0 0.8421 0.8699 0.8557 Height 30.0 10.0 15.0 45.0 0.75 0.6667 0.7059 Frequency 595.0 99.0 138.0 733.0 0.8573 0.8117 0.8339 Route 858.0 76.0 89.0 947.0 0.9186 0.906 0.9123 Duration 351.0 99.0 108.0 459.0 0.78 0.7647 0.7723 Death_Entity 43.0 14.0 5.0 48.0 0.7544 0.8958 0.819 Internal_organ_or... 6477.0 972.0 991.0 7468.0 0.8695 0.8673 0.8684 Alcohol 80.0 18.0 13.0 93.0 0.8163 0.8602 0.8377 Substance_Quantity 6.0 7.0 4.0 10.0 0.4615 0.6 0.5217 Date 498.0 38.0 19.0 517.0 0.9291 0.9632 0.9459 Hyperlipidemia 47.0 3.0 3.0 50.0 0.94 0.94 0.94 Social_History_He... 99.0 7.0 7.0 106.0 0.934 0.934 0.934 Race_Ethnicity 116.0 0.0 0.0 116.0 1.0 1.0 1.0 Imaging_Technique 40.0 18.0 47.0 87.0 0.6897 0.4598 0.5517 Drug_BrandName 859.0 62.0 61.0 920.0 0.9327 0.9337 0.9332 RelativeDate 566.0 124.0 143.0 709.0 0.8203 0.7983 0.8091 Gender 6096.0 80.0 101.0 6197.0 0.987 0.9837 0.9854 Dosage 244.0 31.0 57.0 301.0 0.8873 0.8106 0.8472 Form 234.0 32.0 55.0 289.0 0.8797 0.8097 0.8432 Medical_History_H... 114.0 9.0 10.0 124.0 0.9268 0.9194 0.9231 Birth_Entity 4.0 2.0 3.0 7.0 0.6667 0.5714 0.6154 Substance 59.0 8.0 11.0 70.0 0.8806 0.8429 0.8613 Sexually_Active_o... 5.0 3.0 4.0 9.0 0.625 0.5556 0.5882 Weight 90.0 10.0 21.0 111.0 0.9 0.8108 0.8531 macro - - - - - - 0.8148 micro - - - - - - 0.8788 ``` --- layout: model title: Legal Economic Analysis Document Classifier (EURLEX) author: John Snow Labs name: legclf_economic_analysis_bert date: 2023-03-06 tags: [en, legal, classification, clauses, economic_analysis, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_economic_analysis_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Economic_Analysis or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Economic_Analysis`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_economic_analysis_bert_en_1.0.0_3.0_1678111814559.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_economic_analysis_bert_en_1.0.0_3.0_1678111814559.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_economic_analysis_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Economic_Analysis]| |[Other]| |[Other]| |[Economic_Analysis]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_economic_analysis_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Economic_Analysis 0.88 0.84 0.86 116 Other 0.84 0.88 0.86 111 accuracy - - 0.86 227 macro-avg 0.86 0.86 0.86 227 weighted-avg 0.86 0.86 0.86 227 ``` --- layout: model title: Arabic Bert Embeddings (MARBERT model v2) author: John Snow Labs name: bert_embeddings_MARBERTv2 date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `MARBERTv2` is a Arabic model orginally trained by `UBC-NLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_MARBERTv2_ar_3.4.2_3.0_1649678231280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_MARBERTv2_ar_3.4.2_3.0_1649678231280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_MARBERTv2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|609.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/UBC-NLP/MARBERTv2 - https://aclanthology.org/2021.acl-long.551.pdf - https://github.com/UBC-NLP/marbert - https://doi.org/10.14288/SOCKEYE - https://www.tensorflow.org/tfrc --- layout: model title: English image_classifier_vit_ice_cream ViTForImageClassification from juanfiguera author: John Snow Labs name: image_classifier_vit_ice_cream date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_ice_cream` is a English model originally trained by juanfiguera. ## Predicted Entities `chocolate ice cream`, `vanilla ice cream` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ice_cream_en_4.1.0_3.0_1660170383355.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ice_cream_en_4.1.0_3.0_1660170383355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_ice_cream", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_ice_cream", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_ice_cream| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Detect PHI for Deidentification purposes (Spanish, reduced entities, augmented data) author: John Snow Labs name: ner_deid_generic_augmented date: 2022-02-15 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 8 entities (1 more than the `ner_deid_generic` ner model). This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf dataset, several data augmentation mechanisms and has been augmented with MEDDOCAN Spanish Deidentification corpus (compared to `ner_deid_generic` which does not include it). It's a generalized version of `ner_deid_subentity_augmented`. ## Predicted Entities `CONTACT`, `NAME`, `DATE`, `ID`, `LOCATION`, `PROFESSION`, `AGE`, `SEX` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_es_3.3.4_2.4_1644925864218.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_es_3.3.4_2.4_1644925864218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("word_embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_generic_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] df = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(df).transform(df) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models") .setInputCols(Array("sentence","token")) .setOutputCol("word_embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_generic_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","word_embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val df = Seq(text).toDF("text") val results = pipeline.fit(df).transform(df) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.generic_augmented").predict(""" Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-NAME| | Miguel| I-NAME| | Martínez| I-NAME| | ,| O| | un| B-SEX| | varón| I-SEX| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-LOCATION| | ,| O| | España| B-LOCATION| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19|B-PROFESSION| | el| O| | dia| O| | 14| O| | de| O| | Marzo| O| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| B-LOCATION| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-LOCATION| | San| I-LOCATION| | Carlos| I-LOCATION| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|15.0 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 CONTACT 185.0 3.0 0.0 185.0 0.984 1.0 0.992 NAME 2066.0 138.0 106.0 2172.0 0.9374 0.9512 0.9442 DATE 1017.0 18.0 18.0 1035.0 0.9826 0.9826 0.9826 ORGANIZATION 2468.0 482.0 332.0 2800.0 0.8366 0.8814 0.8584 ID 65.0 5.0 3.0 68.0 0.9286 0.9559 0.942 SEX 678.0 8.0 15.0 693.0 0.9883 0.9784 0.9833 LOCATION 2532.0 358.0 420.0 2952.0 0.8761 0.8577 0.8668 PROFESSION 246.0 9.0 31.0 277.0 0.9647 0.8881 0.9248 AGE 547.0 8.0 9.0 556.0 0.9856 0.9838 0.9847 macro - - - - - - 0.9421 micro - - - - - - 0.9092 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dm256 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dm256` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm256_en_4.3.0_3.0_1675110661031.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm256_en_4.3.0_3.0_1675110661031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dm256","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dm256","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dm256| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|158.7 MB| ## References - https://huggingface.co/google/t5-efficient-base-dm256 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Cyberbullying Classifier author: John Snow Labs name: classifierdl_use_cyberbullying class: ClassifierDLModel language: en nav_key: models repository: public/models date: 03/07/2020 task: Text Classification edition: Spark NLP 2.5.3 spark_version: 2.4 tags: [classifier] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Identify Racism, Sexism or Neutral tweets. {:.h2_title} ## Predicted Entities ``neutral``, ``racism``, ``sexism``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_CYBERBULLYING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN_CYBERBULLYING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_cyberbullying_en_2.5.3_2.4_1593783319298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_cyberbullying_en_2.5.3_2.4_1593783319298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained(lang="en") \ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") document_classifier = ClassifierDLModel.pretrained('classifierdl_use_cyberbullying', 'en') \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") nlpPipeline = Pipeline(stages=[documentAssembler, use, document_classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate('@geeky_zekey Thanks for showing again that blacks are the biggest racists. Blocked') ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val use = UniversalSentenceEncoder.pretrained(lang="en") .setInputCols(Array("document")) .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_use_cyberbullying", "en") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, use, document_classifier)) val data = Seq("@geeky_zekey Thanks for showing again that blacks are the biggest racists. Blocked").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""@geeky_zekey Thanks for showing again that blacks are the biggest racists. Blocked"""] cyberbull_df = nlu.load('classify.cyberbullying.use').predict(text, output_level='document') cyberbull_df[["document", "cyberbullying"]] ```

{:.h2_title} ## Results ```bash +--------------------------------------------------------------------------------------------------------+------------+ |document |class | +--------------------------------------------------------------------------------------------------------+------------+ |@geeky_zekey Thanks for showing again that blacks are the biggest racists. Blocked. | racism | +--------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| | Model Name | classifierdl_use_cyberbullying | | Model Class | ClassifierDLModel | | Spark Compatibility | 2.5.3 | | Spark NLP Compatibility | 2.4 | | License | open source | | Edition | public | | Input Labels | [document, sentence_embeddings] | | Output Labels | [class] | | Language | en | | Upstream Dependencies | tfhub_use | {:.h2_title} ## Data Source This model is trained on cyberbullying detection dataset. https://raw.githubusercontent.com/dhavalpotdar/cyberbullying-detection/master/data/data/data.csv {:.h2_title} ## Benchmarking ```bash precision recall f1-score support none 0.69 1.00 0.81 3245 racism 0.00 0.00 0.00 568 sexism 0.00 0.00 0.00 922 accuracy 0.69 4735 macro avg 0.23 0.33 0.27 4735 weighted avg 0.47 0.69 0.56 4735 ``` --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_small_pretrained_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-small-pretrained-finetuned-squad` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_pretrained_finetuned_squad_en_4.0.0_3.0_1654184786135.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_pretrained_finetuned_squad_en_4.0.0_3.0_1654184786135.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_small_pretrained_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_small_pretrained_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.small_finetuned.by_anas-awadalla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_small_pretrained_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|107.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-small-pretrained-finetuned-squad --- layout: model title: Detect Assertion Status (assertion_jsl) author: John Snow Labs name: assertion_jsl date: 2021-07-24 tags: [licensed, clinical, assertion, en] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 3.1.2 spark_version: 2.4 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The deep neural network architecture for assertion status detection in Spark NLP is based on a BiLSTM framework, and is a modified version of the architecture proposed by Fancellu et.al. (Fancellu, Lopez, and Webber 2016). Its goal is to classify the assertions made on given medical concepts as being present, absent, or possible in the patient, conditionally present in the patient under certain circumstances, hypothetically present in the patient at some future point, and mentioned in the patient report but associated with someoneelse (Uzuner et al. 2011). ## Predicted Entities `Present`, `Absent`, `Possible`, `Planned`, `Someoneelse`, `Past`, `Family`, `None`, `Hypotetical` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_jsl_en_3.1.2_2.4_1627139823450.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_jsl_en_3.1.2_2.4_1627139823450.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("assertion_jsl", "en", "clinical/models") \ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion]) text="""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""" data = spark.createDataFrame([[text]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala ... val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("assertion_jsl", "en", "clinical/models") .setInputCols(Array("sentence", "ner_chunk", "embeddings")) .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.assert.jsl").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash The output is a dataframe with a sentence per row and an `assertion` column containing all of the assertion labels in the sentence. The assertion column also contains assertion character indices, and other metadata. To get only the entity chunks and assertion labels, without the metadata, select `ner_chunk.result` and `assertion.result` from your output dataframe. +-----------------------------------------+-----+---+----------------------------+-------+---------+ |chunk |begin|end|ner_label |sent_id|assertion| +-----------------------------------------+-----+---+----------------------------+-------+---------+ |21-day-old |17 |26 |Age |0 |Family | |Caucasian |28 |36 |Race_Ethnicity |0 |Family | |male |38 |41 |Gender |0 |Family | |for 2 days |48 |57 |Duration |0 |Family | |congestion |62 |71 |Symptom |0 |Present | |mom |75 |77 |Gender |0 |Family | |yellow |99 |104|Modifier |0 |Family | |discharge |106 |114|Symptom |0 |Family | |nares |135 |139|External_body_part_or_region|0 |Family | |she |147 |149|Gender |0 |Family | |mild |168 |171|Modifier |0 |Family | |problems with his breathing while feeding|173 |213|Symptom |0 |Present | |perioral cyanosis |237 |253|Symptom |0 |Absent | |retractions |258 |268|Symptom |0 |Absent | |One day ago |272 |282|RelativeDate |1 |Family | |mom |285 |287|Gender |1 |Family | |Tylenol |345 |351|Drug_BrandName |1 |Family | |Baby |354 |357|Age |2 |Family | |decreased p.o. intake |377 |397|Symptom |2 |Family | |His |400 |402|Gender |3 |Family | +-----------------------------------------+-----+---+----------------------------+-------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_jsl| |Compatibility:|Healthcare NLP 3.1.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with ‘embeddings_clinical’. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash label prec rec f1 Absent 0.970 0.943 0.956 Someoneelse 0.868 0.775 0.819 Planned 0.721 0.754 0.737 Possible 0.852 0.884 0.868 Past 0.811 0.823 0.817 Present 0.833 0.866 0.849 Family 0.872 0.921 0.896 None 0.609 0.359 0.452 Hypothetical 0.722 0.810 0.763 Macro-average 0.888 0.872 0.880 Micro-average 0.908 0.908 0.908 ``` --- layout: model title: English RobertaForQuestionAnswering (from deepset) author: John Snow Labs name: roberta_qa_roberta_large_squad2_hp date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad2-hp` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad2_hp_en_4.0.0_3.0_1655737792807.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad2_hp_en_4.0.0_3.0_1655737792807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_squad2_hp","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_squad2_hp","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.large.by_deepset").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_squad2_hp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/roberta-large-squad2-hp --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Gam) author: John Snow Labs name: distilbert_qa_base_uncased_cuad_finetuned date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-cuad-distilbert` is a English model originally trained by `Gam`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_cuad_finetuned_en_4.3.0_3.0_1672767889486.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_cuad_finetuned_en_4.3.0_3.0_1672767889486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_cuad_finetuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_cuad_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_cuad_finetuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Gam/distilbert-base-uncased-finetuned-cuad-distilbert --- layout: model title: English image_classifier_vit_lawn_weeds ViTForImageClassification from LorenzoDeMattei author: John Snow Labs name: image_classifier_vit_lawn_weeds date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_lawn_weeds` is a English model originally trained by LorenzoDeMattei. ## Predicted Entities `clover`, `dichondra`, `grass` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lawn_weeds_en_4.1.0_3.0_1660171067931.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lawn_weeds_en_4.1.0_3.0_1660171067931.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_lawn_weeds", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_lawn_weeds", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_lawn_weeds| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect radiology concepts (ner_radiology_wip_clinical) author: John Snow Labs name: ner_radiology_wip_clinical_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_radiology_wip_clinical](https://nlp.johnsnowlabs.com/2021/04/01/ner_radiology_wip_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_radiology_wip_clinical_pipeline_en_4.3.0_3.2_1678801944623.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_radiology_wip_clinical_pipeline_en_4.3.0_3.2_1678801944623.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_radiology_wip_clinical_pipeline", "en", "clinical/models") text = '''Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_radiology_wip_clinical_pipeline", "en", "clinical/models") val text = "Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.radiology.clinical_wip.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:--------------------------|-------------:| | 0 | Bilateral | 0 | 8 | Direction | 0.9828 | | 1 | breast | 10 | 15 | BodyPart | 0.8169 | | 2 | ultrasound | 17 | 26 | ImagingTest | 0.6216 | | 3 | ovoid mass | 78 | 87 | ImagingFindings | 0.6917 | | 4 | 0.5 x 0.5 x 0.4 | 113 | 127 | Measurements | 0.91524 | | 5 | cm | 129 | 130 | Units | 0.9987 | | 6 | anteromedial aspect | 163 | 181 | Direction | 0.8241 | | 7 | left | 190 | 193 | Direction | 0.4667 | | 8 | shoulder | 195 | 202 | BodyPart | 0.6349 | | 9 | mass | 210 | 213 | ImagingFindings | 0.9611 | | 10 | isoechoic echotexture | 228 | 248 | ImagingFindings | 0.6851 | | 11 | muscle | 266 | 271 | BodyPart | 0.7805 | | 12 | internal color flow | 294 | 312 | ImagingFindings | 0.5153 | | 13 | benign fibrous tissue | 334 | 354 | ImagingFindings | 0.394867 | | 14 | lipoma | 361 | 366 | Disease_Syndrome_Disorder | 0.9142 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_radiology_wip_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squadv2-recipe-roberta-tokenwise-token-and-step-losses-3-epochs` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs_en_4.3.0_3.0_1674224122519.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs_en_4.3.0_3.0_1674224122519.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_squadv2_recipe_tokenwise_token_and_step_losses_3_epochs| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/squadv2-recipe-roberta-tokenwise-token-and-step-losses-3-epochs --- layout: model title: English BertForQuestionAnswering Tiny Cased model (from M-FAC) author: John Snow Labs name: bert_qa_tiny_finetuned_squadv2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-finetuned-squadv2` is a English model originally trained by `M-FAC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_tiny_finetuned_squadv2_en_4.0.0_3.0_1657188687594.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_tiny_finetuned_squadv2_en_4.0.0_3.0_1657188687594.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tiny_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_tiny_finetuned_squadv2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tiny_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|16.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/M-FAC/bert-tiny-finetuned-squadv2 - https://arxiv.org/pdf/2107.03356.pdf - https://github.com/IST-DASLab/M-FAC --- layout: model title: Detect Units and Measurements in text author: John Snow Labs name: ner_measurements_clinical date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract units and other measurements in reports, prescription and other medical texts using pretrained NER model. ## Predicted Entities `Units`, `Measurements` {:.btn-box} [ [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_en_3.0.0_3.0_1617260795877.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_en_3.0.0_3.0_1617260795877.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_measurements_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_measurements_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.measurements").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_measurements_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: Legal Law Area Prediction Classifier (Italian) author: John Snow Labs name: legclf_law_area_prediction_italian date: 2023-03-29 tags: [it, licensed, classification, legal, tensorflow] task: Text Classification language: it edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model which identifies law area labels(civil_law, penal_law, public_law, social_law) in Italian-based Court Cases. ## Predicted Entities `civil_law`, `penal_law`, `public_law`, `social_law` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_law_area_prediction_italian_it_1.0.0_3.0_1680095983817.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_law_area_prediction_italian_it_1.0.0_3.0_1680095983817.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_multi_cased", "xx")\ .setInputCols(["document"]) \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_law_area_prediction_italian", "it", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, docClassifier ]) df = spark.createDataFrame([["Per questi motivi, il Tribunale federale pronuncia: 1. Nella misura in cui è ammissibile, il ricorso è respinto. 2. Le spese giudiziarie di fr. 1'000.-- sono poste a carico dei ricorrenti. 3. Comunicazione al patrocinatore dei ricorrenti, al Consiglio di Stato, al Gran Consiglio, al Tribunale amministrativo del Cantone Ticino e all'Ufficio federale dello sviluppo territoriale."]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) result.select("text", "category.result").show(truncate=100) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+------------+ | text| result| +----------------------------------------------------------------------------------------------------+------------+ |Per questi motivi, il Tribunale federale pronuncia: 1. Nella misura in cui è ammissibile, il rico...|[public_law]| +----------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_law_area_prediction_italian| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|it| |Size:|22.3 MB| ## References Train dataset available [here](https://huggingface.co/datasets/rcds/legal_criticality_prediction) ## Benchmarking ```bash label precision recall f1-score support civil_law 0.86 0.86 0.86 58 penal_law 0.85 0.82 0.83 55 public_law 0.79 0.79 0.79 52 social_law 0.93 0.96 0.94 68 accuracy - - 0.86 233 macro-avg 0.86 0.86 0.86 233 weighted-avg 0.86 0.86 0.86 233 ``` --- layout: model title: Finnish asr_wav2vec2_large_uralic_voxpopuli_v2_finnish TFWav2Vec2ForCTC from Finnish-NLP author: John Snow Labs name: pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_uralic_voxpopuli_v2_finnish` is a Finnish model originally trained by Finnish-NLP. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_fi_4.2.0_3.0_1664038039563.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_fi_4.2.0_3.0_1664038039563.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_uralic_voxpopuli_v2_finnish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English image_classifier_vit_base_patch16_224_in21k_classify_4scence ViTForImageClassification from HaoHu author: John Snow Labs name: image_classifier_vit_base_patch16_224_in21k_classify_4scence date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_in21k_classify_4scence` is a English model originally trained by HaoHu. ## Predicted Entities `City_road`, `fog`, `rain`, `snow` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_classify_4scence_en_4.1.0_3.0_1660171116662.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_classify_4scence_en_4.1.0_3.0_1660171116662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_in21k_classify_4scence", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_in21k_classify_4scence", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_in21k_classify_4scence| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Detect Living Species author: John Snow Labs name: bert_token_classifier_ner_living_species date: 2022-06-26 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** - The text files were translated from Spanish with a neural machine translation system. - The annotations were translated with the same neural machine translation system. - The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_en_3.5.3_3.0_1656273939035.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_en_3.5.3_3.0_1656273939035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, ner_model, ner_converter ]) data = spark.createDataFrame([["""42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("ner") .setCaseSensitive(True) .setMaxSentenceLength(512) val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, ner_model, ner_converter)) val data = Seq("""42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.living_species.token_bert").predict("""42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL.""") ```

## Results ```bash +-----------------------+-------+ |ner_chunk |label | +-----------------------+-------+ |woman |HUMAN | |bacterial |SPECIES| |Fusarium spp |SPECIES| |patient |HUMAN | |species |SPECIES| |Fusarium solani complex|SPECIES| |antifungals |SPECIES| +-----------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.83 0.96 0.89 2950 B-SPECIES 0.70 0.93 0.80 3129 I-HUMAN 0.73 0.39 0.51 145 I-SPECIES 0.67 0.81 0.74 1166 micro-avg 0.74 0.91 0.82 7390 macro-avg 0.73 0.77 0.73 7390 weighted-avg 0.75 0.91 0.82 7390 ``` --- layout: model title: Pipeline to Resolve ICD-10-CM Codes author: John Snow Labs name: icd10cm_resolver_pipeline date: 2023-04-28 tags: [en, licensed, clinical, resolver, chunk_mapping, pipeline, icd10cm] task: Entity Resolution language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities with their corresponding ICD-10-CM codes. You’ll just feed your text and it will return the corresponding ICD-10-CM codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_resolver_pipeline_en_4.3.2_3.0_1682726202207.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_resolver_pipeline_en_4.3.2_3.0_1682726202207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline resolver_pipeline = PretrainedPipeline("icd10cm_resolver_pipeline", "en", "clinical/models") text = """A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years and anisakiasis. Also, it was reported that fetal and neonatal hemorrhage""" result = resolver_pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val resolver_pipeline = new PretrainedPipeline("icd10cm_resolver_pipeline", "en", "clinical/models") val result = resolver_pipeline.fullAnnotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years and anisakiasis. Also, it was reported that fetal and neonatal hemorrhage""") ``` {:.nlu-block} ```python import nlu nlu.load("en.icd10cm_resolver.pipeline").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years and anisakiasis. Also, it was reported that fetal and neonatal hemorrhage""") ```

## Results ```bash +-----------------------------+---------+------------+ |chunk |ner_chunk|icd10cm_code| +-----------------------------+---------+------------+ |gestational diabetes mellitus|PROBLEM |O24.919 | |anisakiasis |PROBLEM |B81.0 | |fetal and neonatal hemorrhage|PROBLEM |P545 | +-----------------------------+---------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: English XlmRoBertaForQuestionAnswering (from horsbug98) author: John Snow Labs name: xlm_roberta_qa_Part_2_XLM_Model_E1 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Part_2_XLM_Model_E1` is a English model originally trained by `horsbug98`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_Part_2_XLM_Model_E1_en_4.0.0_3.0_1655983522974.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_Part_2_XLM_Model_E1_en_4.0.0_3.0_1655983522974.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_Part_2_XLM_Model_E1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_Part_2_XLM_Model_E1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.tydiqa.xlm_roberta.v2.by_horsbug98").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_Part_2_XLM_Model_E1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|814.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/horsbug98/Part_2_XLM_Model_E1 --- layout: model title: Word Embeddings for Urdu (urduvec_140M_300d) author: John Snow Labs name: urduvec_140M_300d date: 2020-12-01 task: Embeddings language: ur edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [embeddings, ur, open_source] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained using Word2Vec approach on a corpora of 140 Million tokens, has a vocabulary of 100k unique tokens, and gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/urduvec_140M_300d_ur_2.7.0_2.4_1606810614734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/urduvec_140M_300d_ur_2.7.0_2.4_1606810614734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['مجھے سپارک این ایل پی پسند ہے۔']], ["text"])) ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("مجھے سپارک این ایل پی پسند ہے۔").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["مجھے سپارک این ایل پی پسند ہے۔"] urduvec_df = nlu.load('ur.embed.urdu_vec_140M_300d').predict(text, output_level="token") urduvec_df ```

{:.h2_title} ## Results The model gives 300 dimensional Word2Vec feature vector outputs per token. ```bash |Embeddings vector | Tokens |----------------------------------------------------|--------- | [0.15994004905223846, -0.2213257998228073, 0.0... | مجھے | [-0.16085924208164215, -0.12259697169065475, -... | سپارک | [-0.07977486401796341, -0.528775691986084, 0.3... | این | [-0.24136857688426971, -0.15272589027881622, 0... | ایل | [-0.23666366934776306, -0.16016320884227753, 0... | پی | [0.07911433279514313, 0.05598200485110283, 0.0... | پسند ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|urduvec_140M_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|ur| |Case sensitive:|false| |Dimension:|300| ## Data Source The model is imported from http://www.lrec-conf.org/proceedings/lrec2018/pdf/148.pdf --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_768 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_768_zh_4.2.4_3.0_1670021693429.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_768_zh_4.2.4_3.0_1670021693429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|170.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Oncology Pipeline for Biomarkers author: John Snow Labs name: oncology_biomarker_pipeline date: 2022-11-04 tags: [licensed, en, oncology] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.2.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition, Assertion Status and Relation Extraction models to extract information from oncology texts. This pipeline focuses on entities related to biomarkers. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.2.2_3.0_1667581643291.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.2.2_3.0_1667581643291.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.")[0] ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") val result = pipeline.fullAnnotate("""Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.""")(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_biomarker.pipeline").predict("""Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.""") ```

## Results ```bash ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_oncology_biomarker_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Biomarker | ******************** ner_oncology_test_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | Immunohistochemistry | Pathology_Test | | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_biomarker results ******************** | chunk | ner_label | |:-------------------------------|:----------------------| | Immunohistochemistry | Test | | negative | Biomarker_Measurement | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Measurement | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Measurement | | HER2 | Biomarker | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:------------| | Immunohistochemistry | Pathology_Test | Past | | thyroid transcription factor-1 | Biomarker | Present | | napsin A | Biomarker | Present | | ER | Biomarker | Present | | PR | Biomarker | Present | | HER2 | Oncogene | Present | ******************** assertion_oncology_test_binary_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:----------------| | Immunohistochemistry | Pathology_Test | Medical_History | | thyroid transcription factor-1 | Biomarker | Medical_History | | napsin A | Biomarker | Medical_History | | ER | Biomarker | Medical_History | | PR | Biomarker | Medical_History | | HER2 | Oncogene | Medical_History | ******************** re_oncology_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_related_to | | negative | Biomarker_Result | napsin A | Biomarker | is_related_to | | positive | Biomarker_Result | ER | Biomarker | is_related_to | | positive | Biomarker_Result | PR | Biomarker | is_related_to | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_related_to | ******************** re_oncology_granular_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | is_finding_of | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ******************** re_oncology_biomarker_result_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | is_finding_of | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_biomarker_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP for Healthcare 4.2.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - RelationExtractionModel - RelationExtractionModel --- layout: model title: Acquisitions / Subsidiaries Relation Extraction (md, Unidirectional) author: John Snow Labs name: finre_acquisitions_subsidiaries_md date: 2022-11-08 tags: [acquisition, subsidiaries, en, licensed] task: Relation Extraction language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole financial report. Instead: - Split by paragraphs; - Use the `finclf_acquisitions_item` Text Classifier to select only these paragraphs; This model is a `md` model, meaning that the directions in the relations are meaningful: `chunk1` is the source of the relation, `chunk2` is the target. The aim of this model is to retrieve acquisition or subsidiary relationships between Organizations, included when the acquisition was carried out ("was_acquired") and by whom ("was_acquired_by"). Subsidiaries are tagged with the relationship "is_subsidiary_of". ## Predicted Entities `was_acquired`, `was_acquired_by`, `is_subsidiary_of`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINRE_ACQUISITIONS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finre_acquisitions_subsidiaries_md_en_1.0.0_3.0_1667920790547.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finre_acquisitions_subsidiaries_md_en_1.0.0_3.0_1667920790547.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence was_acquired_by ORG 0 13 Whatsapp, Inc. ORG 31 34 Meta 0.9527305 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finre_acquisitions_subsidiaries_md| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.7 MB| ## References In-house annotations on SEC 10K filings and Wikidata ## Benchmarking ```bash label Recall Precision F1 Support is_subsidiary_of 0.583 0.618 0.600 36 other 0.975 0.948 0.961 243 was_acquired 0.836 0.895 0.864 61 was_acquired_by 0.767 0.780 0.773 60 Avg. 0.790 0.810 0.800 406 Weighted-Avg. 0.887 0.885 0.886 406 ``` --- layout: model title: French XlmRoBertaForQuestionAnswering (from saattrupdan) author: John Snow Labs name: xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan date: 2022-06-24 tags: [open_source, question_answering, xlmroberta, fr] task: Question Answering language: fr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlmr-base-texas-squad-fr` is a French model originally trained by `saattrupdan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan_fr_4.0.0_3.0_1656066193139.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan_fr_4.0.0_3.0_1656066193139.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan","fr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan","fr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fr.answer_question.squad.xlmr_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlmr_base_texas_squad_fr_fr_saattrupdan| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fr| |Size:|873.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saattrupdan/xlmr-base-texas-squad-fr --- layout: model title: Sentence Entity Resolver for Snomed (sbertresolve_snomed_conditions) author: John Snow Labs name: sbertresolve_snomed_conditions date: 2021-08-28 tags: [snomed, licensed, en, clinical] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.3 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities (domain: Conditions) to Snomed codes using `sbert_jsl_medium_uncased` Sentence Bert Embeddings. ## Predicted Entities Snomed Codes and their normalized definition with `sbert_jsl_medium_uncased ` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_snomed_conditions_en_3.1.3_2.4_1630180858399.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_snomed_conditions_en_3.1.3_2.4_1630180858399.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained('sbert_jsl_medium_uncased', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("snomed_code")\ .setDistanceFunction("EUCLIDEAN") snomed_pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, snomed_resolver ]) snomed_lp = LightPipeline(snomed_pipelineModel) result = snomed_lp.fullAnnotate("schizophrenia") ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained('sbert_jsl_medium_uncased', 'en','clinical/models') .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") \ .setInputCols(Array("sbert_embeddings")) \ .setOutputCol("snomed_code")\ .setDistanceFunction("EUCLIDEAN") val snomed_pipelineModel = new PipelineModel().setStages(Array(documentAssembler,sbert_embedder,snomed_resolver)) val snomed_lp = LightPipeline(snomed_pipelineModel) val result = snomed_lp.fullAnnotate("schizophrenia") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed_conditions").predict("""Put your text here.""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:--------------|:---------|:-------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------|:-----------------------------------------------------| | 0 | schizophrenia | 58214004 | [schizophrenia, chronic schizophrenia, borderline schizophrenia, schizophrenia, catatonic, subchronic schizophrenia, ...]| [58214004, 83746006, 274952002, 191542003, 191529003, 16990005, ...] | 0.0000, 0.0774, 0.0838, 0.0927, 0.0970, 0.0970, ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_snomed_conditions| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[snomed_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: Arabic Part of Speech Tagger (Modern Standard Arabic (MSA), Modern Standard Arabic-MSA POS) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_msa_pos_msa date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-msa-pos-msa` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_msa_pos_msa_ar_3.4.2_3.0_1650993589088.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_msa_pos_msa_ar_3.4.2_3.0_1650993589088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_msa_pos_msa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-pos-msa - https://dl.acm.org/doi/pdf/10.5555/1621804.1621808 - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: Persian BertForMaskedLM Base Cased model (from HooshvareLab) author: John Snow Labs name: bert_embeddings_fa_zwnj_base date: 2022-12-02 tags: [fa, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: fa edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-fa-zwnj-base` is a Persian model originally trained by `HooshvareLab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_fa_zwnj_base_fa_4.2.4_3.0_1670019440290.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_fa_zwnj_base_fa_4.2.4_3.0_1670019440290.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_fa_zwnj_base","fa") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_fa_zwnj_base","fa") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_fa_zwnj_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fa| |Size:|444.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/HooshvareLab/bert-fa-zwnj-base - https://arxiv.org/abs/2005.12515 - https://github.com/hooshvare/parsbert/issues --- layout: model title: XLM-RoBERTa Base NER Pipeline author: ahmedlone127 name: xlm_roberta_base_token_classifier_ontonotes_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, xlm_roberta, ontonotes, xlm, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_base_token_classifier_ontonotes](https://nlp.johnsnowlabs.com/2021/10/03/xlm_roberta_base_token_classifier_ontonotes_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/xlm_roberta_base_token_classifier_ontonotes_pipeline_en_4.0.0_3.0_1655216428417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/xlm_roberta_base_token_classifier_ontonotes_pipeline_en_4.0.0_3.0_1655216428417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlm_roberta_base_token_classifier_ontonotes_pipeline", lang = "en") pipeline.annotate("My name is John and I have been working at John Snow Labs since November 2020.") ``` ```scala val pipeline = new PretrainedPipeline("xlm_roberta_base_token_classifier_ontonotes_pipeline", lang = "en") pipeline.annotate("My name is John and I have been working at John Snow Labs since November 2020.") ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PERSON | |John Snow Labs|ORG | |November 2020 |DATE | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_base_token_classifier_ontonotes_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|858.4 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - XlmRoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury TFWav2Vec2ForCTC from Satyamatury author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury` is a English model originally trained by Satyamatury. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury_en_4.2.0_3.0_1664112233589.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury_en_4.2.0_3.0_1664112233589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_Satyamatury| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_5 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-5` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_5_en_4.3.0_3.0_1672767558441.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_5_en_4.3.0_3.0_1672767558441.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_5","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_5","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_5| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-5 --- layout: model title: Translate Berber to English Pipeline author: John Snow Labs name: translate_ber_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ber, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ber` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ber_en_xx_2.7.0_2.4_1609687862998.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ber_en_xx_2.7.0_2.4_1609687862998.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ber_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ber_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ber.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ber_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: PICO Classifier author: John Snow Labs name: classifierdl_pico_biobert date: 2020-11-12 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.6.2 spark_version: 2.4 tags: [classifier, en, licensed, clinical] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Classify medical text according to PICO framework. {:.h2_title} ## Predicted Entities ``CONCLUSIONS``, ``DESIGN_SETTING``, ``INTERVENTION``, ``PARTICIPANTS``, ``FINDINGS``, ``MEASUREMENTS``, ``AIMS``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_PICO/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CLINICAL_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_pico_biobert_en_2.6.2_2.4_1601901791781.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_pico_biobert_en_2.6.2_2.4_1601901791781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, BertEmbeddings (biobert_pubmed_base_cased), SentenceEmbeddings, ClassifierDLModel.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained('biobert_pubmed_base_cased')\ .setInputCols(["document", 'token'])\ .setOutputCol("word_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["document", "word_embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifier = ClassifierDLModel.pretrained('classifierdl_pico_biobert', 'en', 'clinical/models')\ .setInputCols(['document', 'token', 'sentence_embeddings']).setOutputCol('class') nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, embeddings, sentence_embeddings, classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate(["""A total of 10 adult daily smokers who reported at least one stressful event and coping episode and provided post-quit data.""", """When carbamazepine is withdrawn from the combination therapy, aripiprazole dose should then be reduced."""]) ``` ```scala ... val embeddings = BertEmbeddings.pretrained('biobert_pubmed_base_cased') .setInputCols(Array("document", 'token')) .setOutputCol("word_embeddings") val sentence_embeddings = SentenceEmbeddings() .setInputCols(Array("document", "word_embeddings")) .setOutputCol("sentence_embeddings") .setPoolingStrategy("AVERAGE") val classifier = ClassifierDLModel.pretrained('classifierdl_pico_biobert', 'en', 'clinical/models') .setInputCols(Array('document', 'token', 'sentence_embeddings')).setOutputCol('class') val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, sentence_embeddings, classifier)) val data = Seq("A total of 10 adult daily smokers who reported at least one stressful event and coping episode and provided post-quit data.", "When carbamazepine is withdrawn from the combination therapy, aripiprazole dose should then be reduced.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.pico").predict("""A total of 10 adult daily smokers who reported at least one stressful event and coping episode and provided post-quit data.""") ```

{:.h2_title} ## Results A dictionary containing class labels for each sentence. ```bash | sentences | class | |------------------------------------------------------+--------------+ | A total of 10 adult daily smokers who reported at... | PARTICIPANTS | | When carbamazepine is withdrawn from the combinat... | CONCLUSIONS | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_pico_biobert| |Type:|ClassifierDLModel| |Compatibility:|Healthcare NLP 2.6.2 +| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|[en]| |Case sensitive:|True| {:.h2_title} ## Data Source Trained on a custom dataset derived from PICO classification dataset, using ``'biobert_pubmed_base_cased'`` embeddings. {:.h2_title} ## Benchmarking ```bash | | labels | precision | recall | f1-score | support | |---:|---------------:|----------:|---------:|---------:|--------:| | 0 | AIMS | 0.9197 | 0.9121 | 0.9159 | 3845 | | 1 | CONCLUSIONS | 0.8426 | 0.8571 | 0.8498 | 4241 | | 2 | DESIGN_SETTING | 0.7703 | 0.8351 | 0.8014 | 5191 | | 3 | FINDINGS | 0.9214 | 0.8964 | 0.9088 | 9500 | | 4 | INTERVENTION | 0.7529 | 0.6758 | 0.7123 | 2597 | | 5 | MEASUREMENTS | 0.8409 | 0.7734 | 0.8058 | 3500 | | 6 | PARTICIPANTS | 0.7521 | 0.8548 | 0.8002 | 2396 | | | accuracy | | 0.8476 | 31270 | | | macro avg | 0.8286 | 0.8292 | 0.8277 | 31270 | | | weighted avg | 0.8495 | 0.8476 | 0.8476 | 31270 | ``` --- layout: model title: Match Chunks in Texts author: John Snow Labs name: match_chunks date: 2022-06-15 tags: [en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The pipeline uses regex `

?/*+` ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/1.SparkNLP_Basics.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/match_chunks_en_4.0.0_3.0_1655322760895.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/match_chunks_en_4.0.0_3.0_1655322760895.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline_local = PretrainedPipeline('match_chunks') result = pipeline_local.annotate("David visited the restaurant yesterday with his family. He also visited and the day before, but at that time he was alone. David again visited today with his colleagues. He and his friends really liked the food and hoped to visit again tomorrow.") result['chunk'] ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline import com.johnsnowlabs.nlp.SparkNLP SparkNLP.version() val testData = spark.createDataFrame(Seq( (1, "David visited the restaurant yesterday with his family. He also visited and the day before, but at that time he was alone. David again visited today with his colleagues. He and his friends really liked the food and hoped to visit again tomorrow."))).toDF("id", "text") val pipeline = PretrainedPipeline("match_chunks", lang="en") val annotation = pipeline.transform(testData) annotation.show() ``` {:.nlu-block} ```python import nlu nlu.load("en.match.chunks").predict("""David visited the restaurant yesterday with his family. He also visited and the day before, but at that time he was alone. David again visited today with his colleagues. He and his friends really liked the food and hoped to visit again tomorrow.""") ```

## Results ```bash ['the restaurant yesterday', 'family', 'the day', 'that time', 'today', 'the food', 'tomorrow'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|match_chunks| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|4.2 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - Chunker --- layout: model title: Word Embeddings for Dutch (dutch_cc_300d) author: John Snow Labs name: dutch_cc_300d date: 2021-10-04 tags: [nl, embeddings, open_source] task: Embeddings language: nl edition: Spark NLP 3.3.0 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia dataset for Dutch language using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dutch_cc_300d_nl_3.3.0_3.0_1633366113070.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dutch_cc_300d_nl_3.3.0_3.0_1633366113070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("dutch_cc_300d", "nl") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) data = spark.createDataFrame([["De Bijlmerramp is de benaming voor de vliegramp"]]).toDF("text") pipeline_model = nlp_pipeline.fit(data) result = pipeline_model.transform(data) ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("dutch_cc_300d", "nl") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("De Bijlmerramp is de benaming voor de vliegramp").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash | token | embedding | |:-------------|:-------------------------------------------------------------------------------| | De | ['0.0249', '-0.0115', '-0.0748', '-0.0823', '0.0866', '-0.0219', '0.00' ...] | | Bijlmerramp | ['0.0204', '0.0079', '0.0224', '0.0352', '-0.0409', '0.0053', '0.0175', ...] | | is | ['-1.0E-4', '0.1419', '0.053', '-0.0921', '0.07', '0.004', '-0.1683', ...] | | de | ['0.0309', '0.0411', '-0.0077', '-0.0756', '0.0741', '-0.0402', '0.025' ...] | | benaming | ['0.0197', '0.0167', '-0.0051', '0.0198', '0.034', '-0.0086', '-0.009', ...] | | voor | ['0.0642', '-0.0171', '-0.0118', '0.0042', '0.0058', '0.0018', '0.0039' ...] | | de | ['0.0309', '0.0411', '-0.0077', '-0.0756', '0.0741', '-0.0402', '0.025' ...] | | vliegramp | ['0.083', '0.025', '0.0029', '0.0064', '-0.0698', '0.0344', '-0.0305', ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dutch_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|nl| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: Detect Assertion Status (DL Large) author: John Snow Labs name: assertion_dl_large_en date: 2020-05-21 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [ner, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deep learning named entity recognition model for assertions. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities ``hypothetical``, ``present``, ``absent``, ``possible``, ``conditional``, ``associated_with_someone_else``. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_large_en_2.5.0_2.4_1590022282256.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_large_en_2.5.0_2.4_1590022282256.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel, NerConverter, AssertionDLModel.

{:.h2_title} ## Results The output is a dataframe with a sentence per row and an ``"assertion"`` column containing all of the assertion labels in the sentence. The assertion column also contains assertion character indices, and other metadata. To get only the entity chunks and assertion labels, without the metadata, select ``"ner_chunk.result"`` and ``"assertion.result"`` from your output dataframe. ```bash chunks entities assertion 0 severe fever PROBLEM present 1 sore throat PROBLEM present 2 stomach pain PROBLEM absent 3 an epidural TREATMENT present 4 PCA TREATMENT present 5 pain control PROBLEM present 6 short of breath PROBLEM conditional 7 CT TEST present 8 lung tumor PROBLEM present 9 Alzheimer PROBLEM associated_with_someone_else ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_dl_large| |Type:|ner| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence, ner_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|[en]| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ {:.h2_title} ## Benchmarking ```bash prec rec f1 absent 0.97 0.91 0.94 associated_with_someone_else 0.93 0.87 0.90 conditional 0.70 0.33 0.44 hypothetical 0.91 0.82 0.86 possible 0.81 0.59 0.68 present 0.93 0.98 0.95 micro avg 0.93 0.93 0.93 macro avg 0.87 0.75 0.80 ``` --- layout: model title: Pipeline to Detect Clinical Entities (bert_token_classifier_ner_clinical) author: John Snow Labs name: bert_token_classifier_ner_clinical_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, berfortokenclassification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_clinical](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_pipeline_en_3.4.1_3.0_1647888696583.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_pipeline_en_3.4.1_3.0_1647888696583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_ner_clinical_pipeline", "en", "clinical/models") pipeline.annotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge .") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_ner_clinical_pipeline", "en", "clinical/models") pipeline.annotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge .") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.clinical_pipeline").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge .""") ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |gestational diabetes mellitus|PROBLEM | |type two diabetes mellitus |PROBLEM | |T2DM |PROBLEM | |HTG-induced pancreatitis |PROBLEM | |an acute hepatitis |PROBLEM | |obesity |PROBLEM | |a body mass index |TEST | |BMI |TEST | |polyuria |PROBLEM | |polydipsia |PROBLEM | |poor appetite |PROBLEM | |vomiting |PROBLEM | |amoxicillin |TREATMENT| |a respiratory tract infection|PROBLEM | |metformin |TREATMENT| |glipizide |TREATMENT| |dapagliflozin |TREATMENT| |T2DM |PROBLEM | |atorvastatin |TREATMENT| |gemfibrozil |TREATMENT| +-----------------------------+---------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverter --- layout: model title: English BertForQuestionAnswering model (from krinal214) author: John Snow Labs name: bert_qa_augmented_Squad_Translated date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `augmented_Squad_Translated` is a English model orginally trained by `krinal214`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_augmented_Squad_Translated_en_4.0.0_3.0_1654179259638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_augmented_Squad_Translated_en_4.0.0_3.0_1654179259638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_augmented_Squad_Translated","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_augmented_Squad_Translated","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.augmented").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_augmented_Squad_Translated| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/krinal214/augmented_Squad_Translated --- layout: model title: Arabic BertForMaskedLM Base Cased model (from aubmindlab) author: John Snow Labs name: bert_embeddings_base_arabertv02 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabertv02` is a Arabic model originally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv02_ar_4.2.4_3.0_1670015827071.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv02_ar_4.2.4_3.0_1670015827071.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv02","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv02","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabertv02| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|507.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-base-arabertv02 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-base-arabertv02/blob/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: English BertForQuestionAnswering Cased model (from DaisyMak) author: John Snow Labs name: bert_qa_finetuned_squad_transformerfrozen_testtoken date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-transformerfrozen-testtoken` is a English model originally trained by `DaisyMak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_finetuned_squad_transformerfrozen_testtoken_en_4.0.0_3.0_1657187107539.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_finetuned_squad_transformerfrozen_testtoken_en_4.0.0_3.0_1657187107539.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_finetuned_squad_transformerfrozen_testtoken","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_finetuned_squad_transformerfrozen_testtoken","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_finetuned_squad_transformerfrozen_testtoken| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/DaisyMak/bert-finetuned-squad-transformerfrozen-testtoken --- layout: model title: Arabic Named Entity Recognition (Dialectal Arabic-DA) author: John Snow Labs name: bert_ner_bert_base_arabic_camelbert_da_ner date: 2022-05-04 tags: [bert, ner, token_classification, ar, open_source] task: Named Entity Recognition language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-da-ner` is a Arabic model orginally trained by `CAMeL-Lab`. ## Predicted Entities `ORG`, `LOC`, `PERS`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_arabic_camelbert_da_ner_ar_3.4.2_3.0_1651630269156.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_arabic_camelbert_da_ner_ar_3.4.2_3.0_1651630269156.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_arabic_camelbert_da_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-da-ner - https://camel.abudhabi.nyu.edu/anercorp/ - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: Part of Speech for Hebrew author: John Snow Labs name: pos_ud_htb date: 2020-12-09 task: Part of Speech Tagging language: he edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [pos, open_source, he] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_htb_he_2.7.0_2.4_1607521333296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_htb_he_2.7.0_2.4_1607521333296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_htb", "he") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate(["ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה"]) ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_htb", "he") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה"] pos_df = nlu.load('he.pos.ud_htb').predict(text, output_level='token') pos_df ```

## Results ```bash {'pos': [Annotation(pos, 0, 0, ADP, {'word': 'ב'}), Annotation(pos, 1, 1, PUNCT, {'word': '-'}), Annotation(pos, 3, 4, NUM, {'word': '25'}), Annotation(pos, 6, 12, VERB, {'word': 'לאוגוסט'}), Annotation(pos, 14, 16, None, {'word': 'עצר'}), Annotation(pos, 18, 22, VERB, {'word': 'השב"כ'}), Annotation(pos, 24, 25, ADP, {'word': 'את'}), Annotation(pos, 27, 31, PROPN, {'word': 'מוחמד'}), Annotation(pos, 33, 42, PROPN, {'word': "אבו-ג'וייד"}), Annotation(pos, 44, 44, PUNCT, {'word': ','}), Annotation(pos, 46, 49, NOUN, {'word': 'אזרח'}), Annotation(pos, 51, 55, ADJ, {'word': 'ירדני'}), Annotation(pos, 57, 57, PUNCT, {'word': ','}), Annotation(pos, 59, 63, VERB, {'word': 'שגויס'}), Annotation(pos, 65, 70, ADP, {'word': 'לארגון'}), Annotation(pos, 72, 76, NOUN, {'word': 'הפת"ח'}), Annotation(pos, 78, 83, PROPN, {'word': 'והופעל'}), Annotation(pos, 85, 86, ADP, {'word': 'על'}), Annotation(pos, 88, 90, NOUN, {'word': 'ידי'}), Annotation(pos, 92, 99, PROPN, {'word': 'חיזבאללה'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_htb| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[tags, document]| |Output Labels:|[pos]| |Language:|he| ## Data Source The model is trained on data obtained from [https://universaldependencies.org](https://universaldependencies.org) ## Benchmarking ```bash | | | precision | recall | f1-score | support | |---:|:-------------|:------------|:---------|-----------:|----------:| | 0 | ADJ | 0.83 | 0.83 | 0.83 | 676 | | 1 | ADP | 0.99 | 0.99 | 0.99 | 1889 | | 2 | ADV | 0.93 | 0.89 | 0.91 | 408 | | 3 | AUX | 0.90 | 0.90 | 0.9 | 229 | | 4 | CCONJ | 0.97 | 0.99 | 0.98 | 434 | | 5 | DET | 0.97 | 0.99 | 0.98 | 1390 | | 6 | NOUN | 0.91 | 0.94 | 0.93 | 3056 | | 7 | NUM | 0.97 | 0.96 | 0.97 | 285 | | 9 | PRON | 0.97 | 0.99 | 0.98 | 443 | | 10 | PROPN | 0.82 | 0.72 | 0.77 | 573 | | 11 | PUNCT | 1.00 | 1.00 | 1 | 1381 | | 12 | SCONJ | 0.99 | 0.90 | 0.94 | 411 | | 13 | VERB | 0.87 | 0.85 | 0.86 | 1063 | | 14 | X | 1.00 | 0.17 | 0.29 | 6 | | 15 | accuracy | | | 0.95 | 15089 | | 16 | macro avg | 0.94 | 0.87 | 0.89 | 15089 | | 17 | weighted avg | 0.95 | 0.95 | 0.95 | 15089 | ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab53_by_hassnain TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab53_by_hassnain date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab53_by_hassnain` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab53_by_hassnain_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab53_by_hassnain_en_4.2.0_3.0_1664022612371.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab53_by_hassnain_en_4.2.0_3.0_1664022612371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab53_by_hassnain", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab53_by_hassnain", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab53_by_hassnain| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Legal Third party beneficiaries Clause Binary Classifier author: John Snow Labs name: legclf_third_party_beneficiaries_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `third-party-beneficiaries` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `third-party-beneficiaries` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_third_party_beneficiaries_clause_en_1.0.0_3.2_1660123110554.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_third_party_beneficiaries_clause_en_1.0.0_3.2_1660123110554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_third_party_beneficiaries_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[third-party-beneficiaries]| |[other]| |[other]| |[third-party-beneficiaries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_third_party_beneficiaries_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-third-party-beneficiaries 0.96 0.96 0.96 49 other 0.98 0.98 0.98 130 accuracy - - 0.98 179 macro-avg 0.97 0.97 0.97 179 weighted-avg 0.98 0.98 0.98 179 ``` --- layout: model title: Mapping Drug Brand Names with Corresponding National Drug Codes author: John Snow Labs name: drug_brandname_ndc_mapper date: 2022-06-26 tags: [chunk_mapper, ndc, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps drug brand names to corresponding National Drug Codes (NDC). Product NDCs for each strength are returned in result and metadata. ## Predicted Entities `Strength_NDC` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/drug_brandname_ndc_mapper_en_3.5.3_3.0_1656260706121.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/drug_brandname_ndc_mapper_en_3.5.3_3.0_1656260706121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("chunk") chunkerMapper = ChunkMapperModel.pretrained("drug_brandname_ndc_mapper", "en", "clinical/models")\ .setInputCols(["chunk"])\ .setOutputCol("ndc")\ .setRels(["Strength_NDC"])\ .setLowerCase(True) pipeline = Pipeline().setStages([ document_assembler, chunkerMapper]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline = LightPipeline(model) result = light_pipeline.fullAnnotate(["zytiga", "zyvana", "ZYVOX"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("chunk") val chunkerMapper = ChunkMapperModel.pretrained("drug_brandname_ndc_mapper", "en", "clinical/models") .setInputCols(Array("chunk")) .setOutputCol("ndc") .setRels(Array("Strength_NDC")) .setLowerCase(True) val pipeline = new Pipeline().setStages(Array( document_assembler, chunkerMapper)) val sample_data = Seq("zytiga", "zyvana", "ZYVOX").toDS.toDF("text") val result = pipeline.fit(sample_data).transform(sample_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.drug_brand_to_ndc").predict("""Put your text here.""") ```

## Results ```bash | | Brandname | Strength_NDC | |---:|:------------|:-------------------------| | 0 | zytiga | 500 mg/1 | 57894-195 | | 1 | zyvana | 527 mg/1 | 69336-405 | | 2 | ZYVOX | 600 mg/300mL | 0009-4992 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|drug_brandname_ndc_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|3.0 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Luba-Katanga author: John Snow Labs name: opus_mt_en_lu date: 2020-12-29 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, lu, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `lu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_lu_xx_2.7.0_2.4_1609281752399.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_lu_xx_2.7.0_2.4_1609281752399.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_lu| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Amharic to Swedish author: John Snow Labs name: opus_mt_am_sv date: 2021-06-01 tags: [open_source, seq2seq, translation, am, sv, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: am target languages: sv {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_am_sv_xx_3.1.0_2.4_1622554609708.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_am_sv_xx_3.1.0_2.4_1622554609708.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_am_sv| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xls_r_thai_test TFWav2Vec2ForCTC from juierror author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_thai_test date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_thai_test` is a English model originally trained by juierror. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_thai_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_thai_test_en_4.2.0_3.0_1664024211955.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_thai_test_en_4.2.0_3.0_1664024211955.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_thai_test', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_thai_test", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_thai_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: ESG Text Classification (3 classes) author: John Snow Labs name: finclf_esg date: 2022-09-06 tags: [en, financial, esg, classification, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model classifies financial texts / news into three classes: Environment, Social and Governance. This model can be use to build a ESG score board for companies. If you look for an augmented version of this model, with more fine-grain verticals (Green House Emissions, Business Ethics, etc), please look for the finance_sequence_classifier_augmented_esg model in Models Hub. ## Predicted Entities `Environment`, `Social`, `Governance` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINCLF_ESG/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_esg_en_1.0.0_3.2_1662472406140.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_esg_en_1.0.0_3.2_1662472406140.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+---------------+ | text| result| +--------------------+---------------+ |The Canadian Envi...|[Environmental]| +--------------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_esg| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|412.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotations from scrapped annual reports and tweets about ESG ## Benchmarking ```bash label precision recall f1-score support Environmental 0.99 0.97 0.98 97 Social 0.95 0.96 0.95 162 Governance 0.91 0.90 0.91 71 accuracy - - 0.95 330 macro-avg 0.95 0.94 0.95 330 weighted-avg 0.95 0.95 0.95 330 ``` --- layout: model title: Mapping Entities with Corresponding ICD-9-CM Codes author: John Snow Labs name: icd9_mapper date: 2022-09-30 tags: [icd9cm, chunk_mapping, en, licensed, clinical] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps entities with their corresponding ICD-9-CM codes. ## Predicted Entities `icd9_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd9_mapper_en_4.1.0_3.0_1664535522949.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd9_mapper_en_4.1.0_3.0_1664535522949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('doc') chunk_assembler = Doc2Chunk()\ .setInputCols(['doc'])\ .setOutputCol('ner_chunk') chunkerMapper = ChunkMapperModel\ .pretrained("icd9_mapper", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["icd9_code"]) mapper_pipeline = Pipeline(stages=[ document_assembler, chunk_assembler, chunkerMapper ]) test_data = spark.createDataFrame([["24 completed weeks of gestation"]]).toDF("text") result = mapper_pipeline.fit(test_data).transform(test_data) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("doc") val chunk_assembler = Doc2Chunk() .setInputCols(Array("doc")) .setOutputCol("ner_chunk") val chunkerMapper = ChunkMapperModel .pretrained("icd9_mapper", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("mappings") .setRels(Array("icd9_code")) val mapper_pipeline = new Pipeline().setStages(Array( document_assembler, chunk_assembler, chunkerMapper)) val test_data = Seq("24 completed weeks of gestation").toDS.toDF("text") val result = mapper_pipeline.fit(test_data).transform(test_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.icd9").predict("""24 completed weeks of gestation""") ```

## Results ```bash +-------------------------------+------------+ |chunk |icd9_mapping| +-------------------------------+------------+ |24 completed weeks of gestation|765.22 | +-------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd9_mapper| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|374.4 KB| --- layout: model title: Entity Recognizer LG author: John Snow Labs name: entity_recognizer_lg date: 2022-06-28 tags: [fi, open_source] task: Named Entity Recognition language: fi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities. It performs most of the common text processing tasks on your dataframe ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_fi_4.0.0_3.0_1656386552101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_fi_4.0.0_3.0_1656386552101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("entity_recognizer_lg", "fi") result = pipeline.annotate("""I love johnsnowlabs! """) ``` {:.nlu-block} ```python import nlu nlu.load("fi.ner.lg").predict("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|2.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becasv2_5 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becasv2-5` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_5_en_4.3.0_3.0_1672767788818.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_5_en_4.3.0_3.0_1672767788818.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_5","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_5","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becasv2_5| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becasv2-5 --- layout: model title: English DistilBertForQuestionAnswering model (from ncduy) author: John Snow Labs name: distilbert_qa_base_cased_distilled_squad_finetuned_squad_test date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-finetuned-squad-test` is a English model originally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_squad_finetuned_squad_test_en_4.0.0_3.0_1654723644873.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_squad_finetuned_squad_test_en_4.0.0_3.0_1654723644873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_squad_finetuned_squad_test","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_squad_finetuned_squad_test","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_cased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_distilled_squad_finetuned_squad_test| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/distilbert-base-cased-distilled-squad-finetuned-squad-test --- layout: model title: Extract Demographic Entities from Oncology Texts author: John Snow Labs name: ner_oncology_demographics_wip date: 2022-09-30 tags: [licensed, clinical, oncology, en, ner, demographics] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts demographic information from oncology texts, including age, gender and smoking status. Definitions of Predicted Entities: - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Gender`: Gender-specific nouns and pronouns (including words such as "him" or "she", and family members such as "father"). - `Race_Ethnicity`: The race and ethnicity categories include racial and national origin or sociocultural groups. - `Smoking_Status`: All mentions of smoking related to the patient or to someone else. ## Predicted Entities `Age`, `Gender`, `Race_Ethnicity`, `Smoking_Status` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_demographics_wip_en_4.0.0_3.0_1664563557899.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_demographics_wip_en_4.0.0_3.0_1664563557899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:------------|:---------------| | 40-year-old | Age | | man | Gender | | smoking | Smoking_Status | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_demographics_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|849.2 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Smoking_Status 43.0 12.0 11.0 54.0 0.78 0.80 0.79 Age 679.0 27.0 17.0 696.0 0.96 0.98 0.97 Race_Ethnicity 44.0 7.0 7.0 51.0 0.86 0.86 0.86 Gender 933.0 14.0 8.0 941.0 0.99 0.99 0.99 macro_avg 1699.0 60.0 43.0 1742.0 0.90 0.91 0.90 micro_avg NaN NaN NaN NaN 0.97 0.98 0.97 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from doc2query) author: John Snow Labs name: t5_yahoo_answers_base_v1 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `yahoo_answers-t5-base-v1` is a English model originally trained by `doc2query`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_yahoo_answers_base_v1_en_4.3.0_3.0_1675158667385.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_yahoo_answers_base_v1_en_4.3.0_3.0_1675158667385.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_yahoo_answers_base_v1","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_yahoo_answers_base_v1","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_yahoo_answers_base_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|1.0 GB| ## References - https://huggingface.co/doc2query/yahoo_answers-t5-base-v1 - https://arxiv.org/abs/1904.08375 - https://cs.uwaterloo.ca/~jimmylin/publications/Nogueira_Lin_2019_docTTTTTquery-v2.pdf - https://arxiv.org/abs/2104.08663 - https://github.com/UKPLab/beir - https://www.sbert.net/examples/unsupervised_learning/query_generation/README.html --- layout: model title: English BertForQuestionAnswering model (from srmukundb) author: John Snow Labs name: bert_qa_srmukundb_bert_base_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad` is a English model orginally trained by `srmukundb`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_srmukundb_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181131257.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_srmukundb_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181131257.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_srmukundb_bert_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_srmukundb_bert_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_srmukundb").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_srmukundb_bert_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/srmukundb/bert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForTokenClassification Cased model (from ml6team) author: John Snow Labs name: distilbert_token_classifier_keyphrase_extraction_inspec date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyphrase-extraction-distilbert-inspec` is a English model originally trained by `ml6team`. ## Predicted Entities `KEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_inspec_en_4.3.1_3.0_1678133894118.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_inspec_en_4.3.1_3.0_1678133894118.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_inspec","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_inspec","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_keyphrase_extraction_inspec| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ml6team/keyphrase-extraction-distilbert-inspec - https://dl.acm.org/doi/10.3115/1119355.1119383 - https://paperswithcode.com/sota?task=Keyphrase+Extraction&dataset=inspec --- layout: model title: Word2Vec Embeddings in Azerbaijani (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, az, open_source] task: Embeddings language: az edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_az_3.4.1_3.0_1647284820457.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_az_3.4.1_3.0_1647284820457.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bem_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Mini Uncased model (from ahujaniharika95) author: John Snow Labs name: bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `minilm-uncased-squad2-finetuned-squad` is a English model originally trained by `ahujaniharika95`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad_en_4.0.0_3.0_1657190376033.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad_en_4.0.0_3.0_1657190376033.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_ahujaniharika95_minilm_uncased_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|124.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ahujaniharika95/minilm-uncased-squad2-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Hiligaynon author: John Snow Labs name: opus_mt_en_hil date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, hil, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `hil` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_hil_xx_2.7.0_2.4_1609168441233.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_hil_xx_2.7.0_2.4_1609168441233.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_hil| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual (English, German) DistilBertForQuestionAnswering model (from ZYW) author: John Snow Labs name: distilbert_qa_en_de_model date: 2022-06-08 tags: [en, de, open_source, distilbert, question_answering, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `en-de-model` is a English model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_en_de_model_xx_4.0.0_3.0_1654728267702.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_en_de_model_xx_4.0.0_3.0_1654728267702.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_en_de_model","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_en_de_model","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.distil_bert.en_de_tuned.by_ZYW").predict("""PUT YOUR QUESTION HERE|||"PUT YOUR CONTEXT HERE""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_en_de_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/en-de-model --- layout: model title: Legal Registration expenses Clause Binary Classifier author: John Snow Labs name: legclf_registration_expenses_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `registration-expenses` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `registration-expenses` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_registration_expenses_clause_en_1.0.0_3.2_1660122893840.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_registration_expenses_clause_en_1.0.0_3.2_1660122893840.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_registration_expenses_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[registration-expenses]| |[other]| |[other]| |[registration-expenses]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_registration_expenses_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.99 0.99 86 registration-expenses 0.97 1.00 0.98 30 accuracy - - 0.99 116 macro-avg 0.98 0.99 0.99 116 weighted-avg 0.99 0.99 0.99 116 ``` --- layout: model title: Legal Subscription Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_subscription_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, subscription, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_subscription_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `subscription-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `subscription-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subscription_agreement_en_1.0.0_3.0_1668111662925.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subscription_agreement_en_1.0.0_3.0_1668111662925.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hjds0923_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hjds0923_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hjds0923_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hjds0923/distilbert-base-uncased-finetuned-squad --- layout: model title: Lemmatizer (French, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, fr] task: Lemmatization language: fr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This French Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_fr_3.4.1_3.0_1646316584024.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_fr_3.4.1_3.0_1646316584024.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------+ |result | +--------------------------------+ |[Tu, n'es, pas, mieux, que, moi]| +--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|fr| |Size:|2.4 MB| --- layout: model title: English asr_Part1 TFWav2Vec2ForCTC from zasheza author: John Snow Labs name: asr_Part1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Part1` is a English model originally trained by zasheza. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Part1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Part1_en_4.2.0_3.0_1664039751422.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Part1_en_4.2.0_3.0_1664039751422.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Part1", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Part1", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Part1| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English BertForQuestionAnswering model (from juliusco) author: John Snow Labs name: bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert-base-cased-v1.1-squad-finetuned-covdrobert` is a English model orginally trained by `juliusco`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert_en_4.0.0_3.0_1654185648282.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert_en_4.0.0_3.0_1654185648282.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.covid_roberta.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_base_cased_v1.1_squad_finetuned_covdrobert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/juliusco/biobert-base-cased-v1.1-squad-finetuned-covdrobert --- layout: model title: Fast Neural Machine Translation Model from Hungarian to English author: John Snow Labs name: opus_mt_hu_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, hu, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `hu` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_hu_en_xx_2.7.0_2.4_1609168984163.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_hu_en_xx_2.7.0_2.4_1609168984163.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_hu_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: BioBERT Embeddings (Pubmed PMC) author: John Snow Labs name: biobert_pubmed_pmc_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pubmed_pmc_base_cased_en_2.6.0_2.4_1598343200280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pubmed_pmc_base_cased_en_2.6.0_2.4_1598343200280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("biobert_pubmed_pmc_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("biobert_pubmed_pmc_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.pubmed_pmc_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_biobert_pubmed_pmc_base_cased_embeddings I [-0.012962102890014648, 0.27699071168899536, 0... hate [0.1688309609889984, 0.5337603688240051, 0.148... cancer [0.1850549429655075, 0.05875205248594284, -0.5... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_pubmed_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_800000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-800000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_800000_cased_generator_de_3.4.4_3.0_1652786505011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_800000_cased_generator_de_3.4.4_3.0_1652786505011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_800000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-800000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: English Bert Embeddings (from nlp4good) author: John Snow Labs name: bert_embeddings_psych_search date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `psych-search` is a English model orginally trained by `nlp4good`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_psych_search_en_3.4.2_3.0_1649672127720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_psych_search_en_3.4.2_3.0_1649672127720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_psych_search| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|412.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/nlp4good/psych-search - https://meshb.nlm.nih.gov/treeView - https://meshb.nlm.nih.gov/record/ui?ui=D000072339 - https://meshb.nlm.nih.gov/record/ui?ui=D005006 - https://meshb.nlm.nih.gov/treeView - http://bioasq.org/ --- layout: model title: English asr_wav2vec2_large_english TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_wav2vec2_large_english date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_english` is a English model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_english_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_english_en_4.2.0_3.0_1664020317451.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_english_en_4.2.0_3.0_1664020317451.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_english', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_english", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_english| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Word2Vec Embeddings in Norwegian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: ["no", open_source] task: Embeddings language: "no" edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_no_3.4.1_3.0_1647448666485.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_no_3.4.1_3.0_1647448666485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|no| |Size:|1.2 GB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Extract Cancer Therapies and Granular Posology Information author: John Snow Labs name: ner_oncology_posology date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner, treatment, posology] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts cancer therapies (Cancer_Surgery, Radiotherapy and Cancer_Therapy) and posology information at a granular level. Definitions of Predicted Entities: - `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. - `Cancer_Therapy`: Any cancer treatment mentioned in text, excluding surgeries and radiotherapy. - `Cycle_Count`: The total number of cycles being administered of an oncological therapy (e.g. "5 cycles"). - `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. "day 5"). - `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. "third cycle"). - `Dosage`: The quantity prescribed by the physician for an active ingredient. - `Duration`: Words indicating the duration of a treatment (e.g. "for 2 weeks"). - `Frequency`: Words indicating the frequency of treatment administration (e.g. "daily" or "bid"). - `Radiotherapy`: Terms that indicate the use of Radiotherapy. - `Radiation_Dose`: Dose used in radiotherapy. - `Route`: Words indicating the type of administration route (such as "PO" or "transdermal"). ## Predicted Entities `Cancer_Surgery`, `Cancer_Therapy`, `Cycle_Count`, `Cycle_Day`, `Cycle_Number`, `Dosage`, `Duration`, `Frequency`, `Radiotherapy`, `Radiation_Dose`, `Route` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_posology_en_4.0.0_3.0_1666728701834.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_posology_en_4.0.0_3.0_1666728701834.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:---------------| | adriamycin | Cancer_Therapy | | 60 mg/m2 | Dosage | | cyclophosphamide | Cancer_Therapy | | 600 mg/m2 | Dosage | | six courses | Cycle_Count | | second cycle | Cycle_Number | | chemotherapy | Cancer_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_posology| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Cycle_Number 52 4 45 97 0.93 0.54 0.68 Cycle_Count 200 63 30 230 0.76 0.87 0.81 Radiotherapy 255 16 55 310 0.94 0.82 0.88 Cancer_Surgery 592 66 227 819 0.90 0.72 0.80 Cycle_Day 175 22 73 248 0.89 0.71 0.79 Frequency 337 44 90 427 0.88 0.79 0.83 Route 53 1 60 113 0.98 0.47 0.63 Cancer_Therapy 1448 81 250 1698 0.95 0.85 0.90 Duration 525 154 236 761 0.77 0.69 0.73 Dosage 858 79 202 1060 0.92 0.81 0.86 Radiation_Dose 86 4 40 126 0.96 0.68 0.80 macro_avg 4581 534 1308 5889 0.90 0.72 0.79 micro_avg 4581 534 1308 5889 0.90 0.78 0.83 ``` --- layout: model title: English asr_english_filipino_wav2vec2_l_xls_r_test_09 TFWav2Vec2ForCTC from Khalsuu author: John Snow Labs name: asr_english_filipino_wav2vec2_l_xls_r_test_09 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_english_filipino_wav2vec2_l_xls_r_test_09` is a English model originally trained by Khalsuu. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_english_filipino_wav2vec2_l_xls_r_test_09_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_english_filipino_wav2vec2_l_xls_r_test_09_en_4.2.0_3.0_1664119314205.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_english_filipino_wav2vec2_l_xls_r_test_09_en_4.2.0_3.0_1664119314205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_english_filipino_wav2vec2_l_xls_r_test_09", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_english_filipino_wav2vec2_l_xls_r_test_09", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_english_filipino_wav2vec2_l_xls_r_test_09| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect Sentences in Healthcare Texts author: John Snow Labs name: sentence_detector_dl_healthcare date: 2021-08-11 tags: [licensed, clinical, en, sentence_detection] task: Sentence Detection language: en nav_key: models edition: Healthcare NLP 3.2.0 spark_version: 3.0 supported: true annotator: SentenceDetectorDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. ## Predicted Entities Breaks text in sentences. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SENTENCE_DETECTOR_HC/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/20.SentenceDetectorDL_Healthcare.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_3.2.0_3.0_1628678815210.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_3.2.0_3.0_1628678815210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl_healthcare","en","clinical/models") \ .setInputCols(["document"]) \ .setOutputCol("sentences") text = """He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. Size: Prostate gland measures 10x1.1x 4.9 cm (LS x AP x TS). Estimated volume is 51.9 ml. , and is mildly enlarged in size.Normal delineation pattern of the prostate gland is preserved. """ sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) result = sd_model.fullAnnotate(text) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","en","clinical/models") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. Size: Prostate gland measures 10x1.1x 4.9 cm (LS x AP x TS). Estimated volume is 51.9 ml. , and is mildly enlarged in size.Normal delineation pattern of the prostate gland is preserved.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.detect_sentence.clinical").predict("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. Size: Prostate gland measures 10x1.1x 4.9 cm (LS x AP x TS). Estimated volume is 51.9 ml. , and is mildly enlarged in size.Normal delineation pattern of the prostate gland is preserved. """) ```

## Results ```bash | | sentences | |---:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety. | | 1 | Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. | | 2 | Size: Prostate gland measures 10x1.1x 4.9 cm (LS x AP x TS). | | 3 | Estimated volume is | | | 51.9 ml. , and is mildly enlarged in size. | | 4 | Normal delineation pattern of the prostate gland is preserved. | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl_healthcare| |Compatibility:|Healthcare NLP 3.2.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|en| ## Data Source Healthcare SDDL model is trained on in-house domain specific data. ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Fast Neural Machine Translation Model from Haitian Creole to English author: John Snow Labs name: opus_mt_ht_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ht, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ht` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ht_en_xx_2.7.0_2.4_1609166270703.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ht_en_xx_2.7.0_2.4_1609166270703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ht_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Sales Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_sales_bert date: 2023-03-05 tags: [en, legal, classification, clauses, sales, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Sales` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Sales`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sales_bert_en_1.0.0_3.0_1678049898342.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sales_bert_en_1.0.0_3.0_1678049898342.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_sales_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Sales]| |[Other]| |[Other]| |[Sales]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sales_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.94 0.89 0.91 54 Sales 0.84 0.91 0.88 35 accuracy - - 0.90 89 macro-avg 0.89 0.90 0.90 89 weighted-avg 0.90 0.90 0.90 89 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `recipe_triplet_recipes-roberta-base_EASY_squadv2_epochs_3` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3_en_4.3.0_3.0_1674212163630.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3_en_4.3.0_3.0_1674212163630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_recipe_triplet_recipes_base_easy_squadv2_epochs_3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/recipe_triplet_recipes-roberta-base_EASY_squadv2_epochs_3 --- layout: model title: English asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent TFWav2Vec2ForCTC from creynier author: John Snow Labs name: pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent` is a English model originally trained by creynier. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_en_4.2.0_3.0_1664042475596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_en_4.2.0_3.0_1664042475596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese Bert Embeddings (Base, Plus, Wobert model) author: John Snow Labs name: bert_embeddings_wobert_chinese_plus_base date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `wobert_chinese_plus_base` is a Chinese model orginally trained by `junnyu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_wobert_chinese_plus_base_zh_3.4.2_3.0_1649669510103.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_wobert_chinese_plus_base_zh_3.4.2_3.0_1649669510103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_wobert_chinese_plus_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|467.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/junnyu/wobert_chinese_plus_base - https://github.com/ZhuiyiTechnology/WoBERT - https://github.com/JunnYu/WoBERT_pytorch --- layout: model title: Smaller BERT Embeddings (L-8_H-128_A-2) author: John Snow Labs name: small_bert_L8_128 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L8_128_en_2.6.0_2.4_1598344352001.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L8_128_en_2.6.0_2.4_1598344352001.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L8_128", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L8_128", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L8_128').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L8_128_embeddings I [1.8417736291885376, 0.29461684823036194, -0.3... love [2.903827428817749, 0.6693897247314453, -0.338... NLP [1.8207342624664307, 0.1299048662185669, -1.94... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L8_128| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|128| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1](https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1) --- layout: model title: Lemmatization from BSC/projecte_aina lookups author: cayorodriguez name: lemmatizer_bsc date: 2022-07-07 tags: [ca, open_source] task: Lemmatization language: ca edition: Spark NLP 3.4.4 spark_version: 3.0 supported: false recommended: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Lemmatizer using lookup tables from `BSC/projecte_aina` sources. This Lemmatizer should work with specific tokenization rules included in the Python usage section. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/cayorodriguez/lemmatizer_bsc_ca_3.4.4_3.0_1657199421685.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/cayorodriguez/lemmatizer_bsc_ca_3.4.4_3.0_1657199421685.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") ex_list = ["aprox\.","pàg\.","p\.ex\.","gen\.","feb\.","abr\.","jul\.","set\.","oct\.","nov\.","dec\.","dr\.","dra\.","sr\.","sra\.","srta\.","núm\.","st\.","sta\.","pl\.","etc\.", "ex\."] ex_list_all = [] ex_list_all.extend(ex_list) ex_list_all.extend([x[0].upper() + x[1:] for x in ex_list]) ex_list_all.extend([x.upper() for x in ex_list]) tokenizer = Tokenizer() \ .setInputCols(['sentence']).setOutputCol('token')\ .setInfixPatterns(["(d|D)(els)","(d|D)(el)","(a|A)(ls)","(a|A)(l)","(p|P)(els)","(p|P)(el)",\ "([A-zÀ-ú_@]+)(-[A-zÀ-ú_@]+)",\ "(d'|D')([·A-zÀ-ú@_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+","(l'|L')([·A-zÀ-ú_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+", \ "(l'|l'|s'|s'|d'|d'|m'|m'|n'|n'|D'|D'|L'|L'|S'|S'|N'|N'|M'|M')([A-zÀ-ú_]+)",\ """([A-zÀ-ú·]+)(\.|,|\)|\?|!|;|\:|\"|”)(\.|,|\)|\?|!|;|\:|\"|”)+""",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|,|;|:|\?|,)+",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|')+([0-9A-zÀ-ú_]+)",\ "([0-9A-zÀ-ú·]+)(\.|\"|;|:|!|\?|$|$|”|“|'|,|%)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+([0-9]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+",\ "(d'|D'|l'|L')([·A-zÀ-ú@_]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)", \ "([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)"]) \ .setExceptions(ex_list_all).fit(data) lemmatizer = LemmatizerModel.pretrained("lemmatizer_bsc","ca") \ .setInputCols(["token"]) \ .setOutputCol("lemma") pipeline = Pipeline(stages=[documentAssembler, tokenizer, lemmatizer]) example = spark.createDataFrame([["Bons dies, al mati"]], ["text"]) results = pipeline.fit(example).transform(example) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemmatizer_bsc| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Community| |Input Labels:|[form]| |Output Labels:|[lemma]| |Language:|ca| |Size:|7.3 MB| --- layout: model title: English asr_Dansk_wav2vec21 TFWav2Vec2ForCTC from Siyam author: John Snow Labs name: pipeline_asr_Dansk_wav2vec21 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Dansk_wav2vec21` is a English model originally trained by Siyam. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Dansk_wav2vec21_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Dansk_wav2vec21_en_4.2.0_3.0_1664118614170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Dansk_wav2vec21_en_4.2.0_3.0_1664118614170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Dansk_wav2vec21', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Dansk_wav2vec21", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Dansk_wav2vec21| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RoBERTa Embeddings (Large, Wikipedia and Bookcorpus datasets) author: John Snow Labs name: roberta_embeddings_muppet_roberta_large date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `muppet-roberta-large` is a English model orginally trained by `facebook`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_muppet_roberta_large_en_3.4.2_3.0_1649946679876.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_muppet_roberta_large_en_3.4.2_3.0_1649946679876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_muppet_roberta_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|849.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/facebook/muppet-roberta-large - https://arxiv.org/abs/2101.11038 --- layout: model title: Portuguese Named Entity Recognition (from m-lin20) author: John Snow Labs name: bert_ner_satellite_instrument_bert_NER date: 2022-05-09 tags: [bert, ner, token_classification, pt, open_source] task: Named Entity Recognition language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `satellite-instrument-bert-NER` is a Portuguese model orginally trained by `m-lin20`. ## Predicted Entities `satellite`, `instrument` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_satellite_instrument_bert_NER_pt_3.4.2_3.0_1652098534939.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_satellite_instrument_bert_NER_pt_3.4.2_3.0_1652098534939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_satellite_instrument_bert_NER| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|pt| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m-lin20/satellite-instrument-bert-NER - https://github.com/Tsinghua-mLin/satellite-instrument-NER --- layout: model title: German T5ForConditionalGeneration Small Cased model (from aiassociates) author: John Snow Labs name: t5_small_grammar_correction date: 2023-01-31 tags: [de, open_source, t5, tensorflow] task: Text Generation language: de edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-grammar-correction-german` is a German model originally trained by `aiassociates`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_grammar_correction_de_4.3.0_3.0_1675126287089.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_grammar_correction_de_4.3.0_3.0_1675126287089.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_grammar_correction","de") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_grammar_correction","de") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_grammar_correction| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|de| |Size:|288.7 MB| ## References - https://huggingface.co/aiassociates/t5-small-grammar-correction-german - https://github.com/EricFillion/happy-transformer - https://www.ai.associates/ - https://www.linkedin.com/company/ai-associates --- layout: model title: Sentence Embeddings - sbert tiny (tuned) author: John Snow Labs name: sbert_jsl_tiny_umls_uncased date: 2021-06-30 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_tiny_umls_uncased_en_3.1.0_2.4_1625050224767.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_tiny_umls_uncased_en_3.1.0_2.4_1625050224767.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings.pretrained("sbert_jsl_tiny_umls_uncased","en","clinical/models").setInputCols(["sentence"]).setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_tiny_umls_uncased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_tiny_umls_uncased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_tiny_umls_uncased| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|false| ## Data Source Tuned on MedNLI and UMLS dataset ## Benchmarking ```bash MedNLI Score Acc 0.616 STS(cos) 0.632 ``` --- layout: model title: Spanish RobertaForQuestionAnswering (from hackathon-pln-es) author: John Snow Labs name: roberta_qa_roberta_base_bne_squad2_hackathon_pln date: 2022-06-21 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-squad2-es` is a Spanish model originally trained by `hackathon-pln-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_bne_squad2_hackathon_pln_es_4.0.0_3.0_1655790288763.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_bne_squad2_hackathon_pln_es_4.0.0_3.0_1655790288763.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_bne_squad2_hackathon_pln","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_bne_squad2_hackathon_pln","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_bne_squad2_hackathon_pln| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|456.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/hackathon-pln-es/roberta-base-bne-squad2-es --- layout: model title: English DistilBertForQuestionAnswering Cased model (from aszidon) author: John Snow Labs name: distilbert_qa_custom date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom_en_4.3.0_3.0_1672774581586.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom_en_4.3.0_3.0_1672774581586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom --- layout: model title: English image_classifier_vit_croupier_creature_classifier ViTForImageClassification from alkzar90 author: John Snow Labs name: image_classifier_vit_croupier_creature_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_croupier_creature_classifier` is a English model originally trained by alkzar90. ## Predicted Entities `elf`, `goblin`, `knight`, `zombie` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_croupier_creature_classifier_en_4.1.0_3.0_1660171498624.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_croupier_creature_classifier_en_4.1.0_3.0_1660171498624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_croupier_creature_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_croupier_creature_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_croupier_creature_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Entity Resolver for Human Phenotype Ontology author: John Snow Labs name: sbiobertresolve_HPO date: 2021-05-05 tags: [en, licensed, clinical, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps phenotypic abnormalities encountered in human diseases to Human Phenotype Ontology (HPO) codes. ## Predicted Entities This model returns Human Phenotype Ontology (HPO) codes for phenotypic abnormalities encountered in human diseases. It also returns associated codes from the following vocabularies for each HPO code: - MeSH (Medical Subject Headings) - SNOMED - UMLS (Unified Medical Language System ) - ORPHA (international reference resource for information on rare diseases and orphan drugs) - OMIM (Online Mendelian Inheritance in Man) {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_HPO/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_HPO_en_3.0.2_3.0_1620235451661.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_HPO_en_3.0.2_3.0_1620235451661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_HPO``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_human_phenotype_gene_clinical``` as NER model. No need to ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli",'en','clinical/models')\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_HPO", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline = Pipeline(stages = [document_assembler, sentence_detector, tokens, embeddings, ner, ner_converter, chunk2doc, sbert_embedder, resolver]) model = LightPipeline(pipeline.fit(spark.createDataFrame([['']], ["text"]))) text="""These disorders include cancer, bipolar disorder, schizophrenia, autism, Cri-du-chat syndrome, myopia, cortical cataract-linked Alzheimer's disease, and infectious diseases""" results = model.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.HPO").predict("""These disorders include cancer, bipolar disorder, schizophrenia, autism, Cri-du-chat syndrome, myopia, cortical cataract-linked Alzheimer's disease, and infectious diseases""") ```

## Results ```bash | | chunk | entity | resolution | aux_codes | |---:|:-----------------|:---------|:-------------|:-----------------------------------------------------------------------------| | 0 | cancer | HP | HP:0002664 | MSH:D009369||SNOMED:108369006,363346000||UMLS:C0006826,C0027651||ORPHA:1775 | | 1 | bipolar disorder | HP | HP:0007302 | MSH:D001714||SNOMED:13746004||UMLS:C0005586||ORPHA:370079 | | 2 | schizophrenia | HP | HP:0100753 | MSH:D012559||SNOMED:191526005,58214004||UMLS:C0036341||ORPHA:231169 | | 3 | autism | HP | HP:0000717 | MSH:D001321||SNOMED:408856003,408857007,43614003||UMLS:C0004352||ORPHA:79279 | | 4 | myopia | HP | HP:0000545 | MSH:D009216||SNOMED:57190000||UMLS:C0027092||ORPHA:370022 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_HPO| |Compatibility:|Healthcare NLP 3.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[hpo_code]| |Language:|en| --- layout: model title: Legal Exclusivity Clause Binary Classifier author: John Snow Labs name: legclf_exclusivity_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `exclusivity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `exclusivity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_exclusivity_clause_en_1.0.0_3.2_1660122411802.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_exclusivity_clause_en_1.0.0_3.2_1660122411802.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_exclusivity_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[exclusivity]| |[other]| |[other]| |[exclusivity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_exclusivity_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support exclusivity 0.79 0.61 0.69 36 other 0.86 0.93 0.90 92 accuracy - - 0.84 128 macro-avg 0.82 0.77 0.79 128 weighted-avg 0.84 0.84 0.84 128 ``` --- layout: model title: English BertForQuestionAnswering model (from batterydata) author: John Snow Labs name: bert_qa_batterydata_bert_base_uncased_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squad-v1` is a English model orginally trained by `batterydata`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_batterydata_bert_base_uncased_squad_v1_en_4.0.0_3.0_1654181357717.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_batterydata_bert_base_uncased_squad_v1_en_4.0.0_3.0_1654181357717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_batterydata_bert_base_uncased_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_batterydata_bert_base_uncased_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_battery.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_batterydata_bert_base_uncased_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/batterydata/bert-base-uncased-squad-v1 - https://github.com/ShuHuang/batterybert --- layout: model title: English asr_wav2vec2_xls_r_300m_kh TFWav2Vec2ForCTC from kongkeaouch author: John Snow Labs name: asr_wav2vec2_xls_r_300m_kh date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_kh` is a English model originally trained by kongkeaouch. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_kh_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_kh_en_4.2.0_3.0_1664025079738.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_kh_en_4.2.0_3.0_1664025079738.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_kh", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_kh", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_kh| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Legal Representations Clause Binary Classifier author: John Snow Labs name: legclf_representations_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `representations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `representations` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_representations_clause_en_1.0.0_3.2_1660122946365.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_representations_clause_en_1.0.0_3.2_1660122946365.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_representations_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[representations]| |[other]| |[other]| |[representations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_representations_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.98 0.98 645 representations 0.95 0.94 0.95 212 accuracy - - 0.97 857 macro-avg 0.97 0.96 0.96 857 weighted-avg 0.97 0.97 0.97 857 ``` --- layout: model title: Norwegian BertForTokenClassification Base Cased model (from Kushtrim) author: John Snow Labs name: bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner date: 2022-11-30 tags: ["no", open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: "no" edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-norsk-ner` is a Norwegian model originally trained by `Kushtrim`. ## Predicted Entities `MISC`, `LOC`, `ORG`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner_no_4.2.4_3.0_1669815034218.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner_no_4.2.4_3.0_1669815034218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner","no") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner","no") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_base_multilingual_cased_finetuned_norsk_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|no| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Kushtrim/bert-base-multilingual-cased-finetuned-norsk-ner --- layout: model title: Pipeline to Extract Neurologic Deficits Related to Stroke Scale (NIHSS) author: John Snow Labs name: ner_nihss_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: [Named Entity Recognition, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_nihss](https://nlp.johnsnowlabs.com/2021/11/15/ner_nihss_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_NIHSS/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_NIHSS.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_nihss_pipeline_en_3.4.1_3.0_1647871076449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_nihss_pipeline_en_3.4.1_3.0_1647871076449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_nihss_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Abdomen , soft , nontender . NIH stroke scale on presentation was 23 to 24 for , one for consciousness , two for month and year and two for eye / grip , one to two for gaze , two for face , eight for motor , one for limited ataxia , one to two for sensory , three for best language and two for attention . On the neurologic examination the patient was intermittently.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_nihss_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Abdomen , soft , nontender . NIH stroke scale on presentation was 23 to 24 for , one for consciousness , two for month and year and two for eye / grip , one to two for gaze , two for face , eight for motor , one for limited ataxia , one to two for sensory , three for best language and two for attention . On the neurologic examination the patient was intermittently.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.nihss_pipeline").predict("""Abdomen , soft , nontender . NIH stroke scale on presentation was 23 to 24 for , one for consciousness , two for month and year and two for eye / grip , one to two for gaze , two for face , eight for motor , one for limited ataxia , one to two for sensory , three for best language and two for attention . On the neurologic examination the patient was intermittently.""") ```

## Results ```bash | | chunk | entity | |---:|:-------------------|:-------------------------| | 0 | NIH stroke scale | NIHSS | | 1 | 23 to 24 | Measurement | | 2 | one | Measurement | | 3 | consciousness | 1a_LOC | | 4 | two | Measurement | | 5 | month and year | 1b_LOCQuestions | | 6 | two | Measurement | | 7 | eye / grip | 1c_LOCCommands | | 8 | one | Measurement | | 9 | two | Measurement | | 10 | gaze | 2_BestGaze | | 11 | two | Measurement | | 12 | face | 4_FacialPalsy | | 13 | eight | Measurement | | 14 | one | Measurement | | 15 | limited ataxia | 7_LimbAtaxia | | 16 | one to two | Measurement | | 17 | sensory | 8_Sensory | | 18 | three | Measurement | | 19 | best language | 9_BestLanguage | | 20 | two | Measurement | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_nihss_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Stop Words Cleaner for Portuguese author: John Snow Labs name: stopwords_pt date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: pt edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, pt] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_pt_pt_2.5.4_2.4_1594742441703.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_pt_pt_2.5.4_2.4_1594742441703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_pt", "pt") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_pt", "pt") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica."""] stopword_df = nlu.load('pt.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=14, end=16, result='rei', metadata={'sentence': '0'}), Row(annotatorType='token', begin=21, end=25, result='norte', metadata={'sentence': '0'}), Row(annotatorType='token', begin=26, end=26, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=28, end=31, result='John', metadata={'sentence': '0'}), Row(annotatorType='token', begin=33, end=36, result='Snow', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_pt| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|pt| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English DistilBertForQuestionAnswering Cased model (from aszidon) author: John Snow Labs name: distilbert_qa_custom2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom2` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom2_en_4.3.0_3.0_1672774614830.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom2_en_4.3.0_3.0_1672774614830.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom2 --- layout: model title: Spanish BertForQuestionAnswering model (from MMG) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac date: 2022-06-02 tags: [es, open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-spa-squad2-es-finetuned-sqac` is a Spanish model orginally trained by `MMG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac_es_4.0.0_3.0_1654180469657.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac_es_4.0.0_3.0_1654180469657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2_sqac.bert.base_cased_spa.by_MMG").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_finetuned_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MMG/bert-base-spanish-wwm-cased-finetuned-spa-squad2-es-finetuned-sqac --- layout: model title: Legal Tax returns Clause Binary Classifier author: John Snow Labs name: legclf_tax_returns_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `tax-returns` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `tax-returns` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_tax_returns_clause_en_1.0.0_3.2_1660123065637.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_tax_returns_clause_en_1.0.0_3.2_1660123065637.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_tax_returns_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[tax-returns]| |[other]| |[other]| |[tax-returns]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_tax_returns_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash precision recall f1-score support other 1.00 0.92 0.96 37 tax-returns 0.75 1.00 0.86 9 accuracy - - 0.93 46 macro-avg 0.88 0.96 0.91 46 weighted-avg 0.95 0.93 0.94 46 ``` --- layout: model title: Pipeline to Detect Units and Measurements in text author: John Snow Labs name: ner_measurements_clinical_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_measurements_clinical](https://nlp.johnsnowlabs.com/2021/04/01/ner_measurements_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_pipeline_en_4.3.0_3.2_1678832259909.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_pipeline_en_4.3.0_3.2_1678832259909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_measurements_clinical_pipeline", "en", "clinical/models") text = '''Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_measurements_clinical_pipeline", "en", "clinical/models") val text = "Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical_measurements.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------|--------:|------:|:-------------|-------------:| | 0 | 0.5 x 0.5 x 0.4 | 113 | 127 | Measurements | 0.98748 | | 1 | cm | 129 | 130 | Units | 0.9996 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_measurements_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from East Slavic Languages to English author: John Snow Labs name: opus_mt_zle_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, zle, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `zle` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_zle_en_xx_2.7.0_2.4_1609166964113.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_zle_en_xx_2.7.0_2.4_1609166964113.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_zle_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Financial News Summarization (X-Small) author: John Snow Labs name: finsum_news_xs date: 2022-11-23 tags: [financial, summarization, en, licensed] task: Summarization language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Financial news Summarizer, finetuned with a extra-small financial dataset. (about 4K news). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finsum_news_xs_en_1.0.0_3.0_1669213220483.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finsum_news_xs_en_1.0.0_3.0_1669213220483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = nlp.T5Transformer() \ .pretrained("finsum_news_xs" ,"en", "finance/models") \ .setTask("summarize:")\ # or 'summarization' .setMaxOutputLength(512)\ .setInputCols(["documents"]) \ .setOutputCol("summaries") data_df = spark.createDataFrame([["Deere Grows Sales 37% as Shipments Rise. Farm equipment supplier forecasts higher sales in year ahead, lifted by price increases and infrastructure investments. Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more of its farm and construction equipment. The Moline, Ill.-based company, the largest supplier of farm equipment in the U.S., said demand held up as it raised prices on farm equipment, and forecast sales gains in the year ahead. Chief Executive John May cited strong demand and increased investment in infrastructure projects as the Biden administration ramps up spending. Elevated crop prices have kept farmers interested in new machinery even as their own production expenses increase."]]).toDF("text") pipeline = nlp.Pipeline().setStages([document_assembler, t5]) results = pipeline.fit(data_df).transform(data_df) results.select("summaries.result").show(truncate=False) ```

## Results ```bash Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more farm and construction equipment. Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more farm and construction equipment. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finsum_news_xs| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[summaries]| |Language:|en| |Size:|923.2 MB| ## Benchmarking ```bash John Snow Labs in-house summarized articles. ``` --- layout: model title: English DistilBertForQuestionAnswering model (from Thitaree) author: John Snow Labs name: distilbert_qa_Thitaree_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Thitaree`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Thitaree_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724750492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Thitaree_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724750492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Thitaree_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Thitaree_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Thitaree").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Thitaree_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Thitaree/distilbert-base-uncased-finetuned-squad --- layout: model title: Polish BertForQuestionAnswering model (from henryk) author: John Snow Labs name: bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2 date: 2022-06-02 tags: [pl, open_source, question_answering, bert] task: Question Answering language: pl edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-polish-squad2` is a Polish model orginally trained by `henryk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2_pl_4.0.0_3.0_1654180123880.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2_pl_4.0.0_3.0_1654180123880.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2","pl") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2","pl") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("pl.answer_question.squadv2.bert.multilingual_base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_cased_finetuned_polish_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|pl| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/henryk/bert-base-multilingual-cased-finetuned-polish-squad2 - https://www.linkedin.com/in/henryk-borzymowski-0755a2167/ - https://rajpurkar.github.io/SQuAD-explorer/ - https://github.com/google-research/bert/blob/master/multilingual.md --- layout: model title: Finnish asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP TFWav2Vec2ForCTC from Finnish-NLP author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP` is a Finnish model originally trained by Finnish-NLP. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP_fi_4.2.0_3.0_1664040198920.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP_fi_4.2.0_3.0_1664040198920.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_300m_finnish_lm_by_Finnish_NLP| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Oromo to English Pipeline author: John Snow Labs name: translate_om_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, om, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `om` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_om_en_xx_2.7.0_2.4_1609690273713.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_om_en_xx_2.7.0_2.4_1609690273713.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_om_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_om_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.om.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_om_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Anatomical Structures (Single Entity - biobert) author: John Snow Labs name: ner_anatomy_coarse_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_anatomy_coarse_biobert](https://nlp.johnsnowlabs.com/2021/03/31/ner_anatomy_coarse_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_pipeline_en_4.3.0_3.2_1679316528376.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_pipeline_en_4.3.0_3.2_1679316528376.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_anatomy_coarse_biobert_pipeline", "en", "clinical/models") text = '''content in the lung tissue''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_anatomy_coarse_biobert_pipeline", "en", "clinical/models") val text = "content in the lung tissue" val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy_coarse_biobert.pipeline").predict("""content in the lung tissue""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------|--------:|------:|:------------|-------------:| | 0 | lung tissue | 15 | 25 | Anatomy | 0.99155 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_coarse_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from English to Malagasy author: John Snow Labs name: opus_mt_en_mg date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, mg, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `mg` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_mg_xx_2.7.0_2.4_1609167829744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_mg_xx_2.7.0_2.4_1609167829744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_mg| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from avioo1) author: John Snow Labs name: roberta_qa_avioo1_base_squad2_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `avioo1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_avioo1_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219191405.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_avioo1_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219191405.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_avioo1_base_squad2_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_avioo1_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_avioo1_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/avioo1/roberta-base-squad2-finetuned-squad --- layout: model title: Pipeline to Detect Drug Information (Small) author: John Snow Labs name: ner_posology_small_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_posology_small](https://nlp.johnsnowlabs.com/2021/03/31/ner_posology_small_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_pipeline_en_4.3.0_3.2_1678868910811.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_pipeline_en_4.3.0_3.2_1678868910811.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_posology_small_pipeline", "en", "clinical/models") text = '''The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_posology_small_pipeline", "en", "clinical/models") val text = "The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology_small.pipeline").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:---------------|--------:|------:|:------------|-------------:| | 0 | insulin | 59 | 65 | DRUG | 0.9984 | | 1 | Bactrim | 346 | 352 | DRUG | 0.9999 | | 2 | for 14 days | 354 | 364 | DURATION | 0.8678 | | 3 | Fragmin | 925 | 931 | DRUG | 0.9994 | | 4 | 5000 units | 933 | 942 | DOSAGE | 0.88505 | | 5 | subcutaneously | 944 | 957 | ROUTE | 0.9998 | | 6 | daily | 959 | 963 | FREQUENCY | 0.997 | | 7 | Xenaderm | 966 | 973 | DRUG | 0.99 | | 8 | topically | 985 | 993 | ROUTE | 0.971 | | 9 | b.i.d., | 995 | 1001 | FREQUENCY | 0.76095 | | 10 | Lantus | 1003 | 1008 | DRUG | 0.9994 | | 11 | 40 units | 1010 | 1017 | DOSAGE | 0.89935 | | 12 | subcutaneously | 1019 | 1032 | ROUTE | 0.9992 | | 13 | at bedtime | 1034 | 1043 | FREQUENCY | 0.83435 | | 14 | OxyContin | 1046 | 1054 | DRUG | 0.9992 | | 15 | 30 mg | 1056 | 1060 | STRENGTH | 0.99965 | | 16 | p.o | 1062 | 1064 | ROUTE | 0.9997 | | 17 | q.12 h., | 1067 | 1074 | FREQUENCY | 0.9102 | | 18 | folic acid | 1076 | 1085 | DRUG | 0.9955 | | 19 | 1 mg | 1087 | 1090 | STRENGTH | 0.99875 | | 20 | daily | 1092 | 1096 | FREQUENCY | 0.9993 | | 21 | levothyroxine | 1099 | 1111 | DRUG | 0.9998 | | 22 | 0.1 mg | 1113 | 1118 | STRENGTH | 0.99965 | | 23 | p.o | 1120 | 1122 | ROUTE | 0.999 | | 24 | daily | 1125 | 1129 | FREQUENCY | 0.9373 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_small_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: German Medical Bert Embeddings author: John Snow Labs name: bert_embeddings_German_MedBERT date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained German Medical Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `German-MedBERT` is a German model orginally trained by `smanjil`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_German_MedBERT_de_3.4.2_3.0_1649675767629.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_German_MedBERT_de_3.4.2_3.0_1649675767629.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_German_MedBERT| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|409.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/smanjil/German-MedBERT - https://opus4.kobv.de/opus4-rhein-waal/frontdoor/index/index/searchtype/collection/id/16225/start/0/rows/10/doctypefq/masterthesis/docId/740 - https://www.linkedin.com/in/manjil-shrestha-038527b4/ --- layout: model title: French CamemBert Embeddings (from DoyyingFace) author: John Snow Labs name: camembert_embeddings_DoyyingFace_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `DoyyingFace`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_DoyyingFace_generic_model_fr_3.4.4_3.0_1653986003984.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_DoyyingFace_generic_model_fr_3.4.4_3.0_1653986003984.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_DoyyingFace_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/DoyyingFace/dummy-model --- layout: model title: Hindi Named Entity Recognition (from sagorsarker) author: John Snow Labs name: bert_ner_codeswitch_hineng_ner_lince date: 2022-05-09 tags: [bert, ner, token_classification, hi, open_source] task: Named Entity Recognition language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `codeswitch-hineng-ner-lince` is a Hindi model orginally trained by `sagorsarker`. ## Predicted Entities `PERSON`, `ORGANISATION`, `PLACE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_hineng_ner_lince_hi_3.4.2_3.0_1652097576639.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_codeswitch_hineng_ner_lince_hi_3.4.2_3.0_1652097576639.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_codeswitch_hineng_ner_lince| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|hi| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sagorsarker/codeswitch-hineng-ner-lince - https://ritual.uh.edu/lince/home - https://github.com/sagorbrur/codeswitch --- layout: model title: English image_classifier_vit_rust_image_classification_8 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_8 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_8` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_8_en_4.1.0_3.0_1660166811431.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_8_en_4.1.0_3.0_1660166811431.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_8", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_8", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_8| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Sentence Entity Resolver for ICD10-CM (Augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_augmented date: 2021-10-31 tags: [icd10cm, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it has been augmented with synonyms for making it more accurate. ## Predicted Entities `ICD10CM Codes` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_2.4_1635684621243.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_3.3.1_2.4_1635684621243.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk| entity|icd10cm_code| resolutions| all_codes| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O2441|gestational diabetes mellitus:::postpartum gestational diabetes mel...| O2441:::O2443:::Z8632:::Z875:::O2431:::O2411:::O244:::O241:::O2481| |subsequent type two diabetes mellitus|PROBLEM| O2411|pre-existing type 2 diabetes mellitus:::disorder associated with ty...|O2411:::E118:::E11:::E139:::E119:::E113:::E1144:::Z863:::Z8639:::E1...| | T2DM|PROBLEM| E11|type 2 diabetes mellitus:::disorder associated with type 2 diabetes...|E11:::E118:::E119:::O2411:::E109:::E139:::E113:::E8881:::Z833:::D64...| | HTG-induced pancreatitis|PROBLEM| K8520|alcohol-induced pancreatitis:::drug-induced acute pancreatitis:::he...|K8520:::K853:::K8590:::F102:::K852:::K859:::K8580:::K8591:::K858:::...| | acute hepatitis|PROBLEM| K720|acute hepatitis:::acute hepatitis a:::acute infectious hepatitis:::...|K720:::B15:::B179:::B172:::Z0389:::B159:::B150:::B16:::K752:::K712:...| | obesity|PROBLEM| E669|obesity:::abdominal obesity:::obese:::central obesity:::overweight ...|E669:::E668:::Z6841:::Q130:::E66:::E6601:::Z8639:::E349:::H3550:::Z...| | a body mass index|PROBLEM| Z6841|finding of body mass index:::observation of body mass index:::mass ...|Z6841:::E669:::R229:::Z681:::R223:::R221:::Z68:::R222:::R220:::R418...| | polyuria|PROBLEM| R35|polyuria:::nocturnal polyuria:::polyuric state:::polyuric state (di...|R35:::R3581:::R358:::E232:::R31:::R350:::R8299:::N401:::E723:::O048...| | polydipsia|PROBLEM| R631|polydipsia:::psychogenic polydipsia:::primary polydipsia:::psychoge...|R631:::F6389:::E232:::F639:::O40:::G475:::M7989:::R632:::R061:::H53...| | poor appetite|PROBLEM| R630|poor appetite:::poor feeding:::bad taste in mouth:::unpleasant tast...|R630:::P929:::R438:::R432:::E86:::R196:::F520:::Z724:::R0689:::Z768...| | vomiting|PROBLEM| R111|vomiting:::intermittent vomiting:::vomiting symptoms:::periodic vom...| R111:::R11:::R1110:::G43A1:::P921:::P9209:::G43A:::R1113:::R110| | a respiratory tract infection|PROBLEM| J988|respiratory tract infection:::upper respiratory tract infection:::b...|J988:::J069:::A499:::J22:::J209:::Z593:::T17:::J0410:::Z1383:::J189...| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on ICD10CM 2022 Codes dataset: https://www.cdc.gov/nchs/icd/icd10cm.htm --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from it5) author: John Snow Labs name: t5_it5_efficient_small_el32_headline_generation date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-el32-headline-generation` is a Italian model originally trained by `it5`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_headline_generation_it_4.3.0_3.0_1675103295731.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_headline_generation_it_4.3.0_3.0_1675103295731.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_headline_generation","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_headline_generation","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_el32_headline_generation| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|594.0 MB| ## References - https://huggingface.co/it5/it5-efficient-small-el32-headline-generation - https://github.com/stefan-it - https://arxiv.org/abs/2203.03759 - https://gsarti.com - https://malvinanissim.github.io - https://arxiv.org/abs/2109.10686 - https://github.com/gsarti/it5 - https://paperswithcode.com/sota?task=Headline+generation&dataset=HeadGen-IT --- layout: model title: Recognize Entities DL pipeline for Italian - Large author: John Snow Labs name: entity_recognizer_lg date: 2021-03-23 tags: [open_source, italian, entity_recognizer_lg, pipeline, it] supported: true task: [Named Entity Recognition, Lemmatization] language: it edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_it_3.0.0_3.0_1616465464186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_it_3.0.0_3.0_1616465464186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_lg', lang = 'it') annotations = pipeline.fullAnnotate(""Ciao da John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_lg", lang = "it") val result = pipeline.fullAnnotate("Ciao da John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Ciao da John Snow Labs! ""] result_df = nlu.load('it.ner.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Ciao da John Snow Labs! '] | ['Ciao da John Snow Labs!'] | ['Ciao', 'da', 'John', 'Snow', 'Labs!'] | [[-0.238279998302459,.,...]] | ['O', 'O', 'I-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|it| --- layout: model title: Translate Punjabi (Eastern) to English Pipeline author: John Snow Labs name: translate_pa_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, pa, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `pa` - target languages: `en` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/INDIAN_TRANSLATION_PUNJABI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TRANSLATION_PIPELINES_MODELS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_pa_en_xx_2.7.0_2.4_1609690246774.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_pa_en_xx_2.7.0_2.4_1609690246774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline result = pipeline = PretrainedPipeline("translate_pa_en", lang = "xx") pipeline.annotate("ਮੈਨੂੰ ਪੜ੍ਹਨਾ ਪਸੰਦ ਹੈ.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_pa_en", lang = "xx") val result = pipeline.annotate("ਮੈਨੂੰ ਪੜ੍ਹਨਾ ਪਸੰਦ ਹੈ.") ``` {:.nlu-block} ```python import nlu text = ["ਮੈਨੂੰ ਪੜ੍ਹਨਾ ਪਸੰਦ ਹੈ."] translate_df = nlu.load('xx.pa.translate_to.en').predict(text, output_level='sentence') translate_df ```

## Results ```bash +------------------------------+---------------------------+ |sentence |translation | +------------------------------+---------------------------+ |ਮੈਨੂੰ ਪੜ੍ਹਨਾ ਪਸੰਦ ਹੈ. |I like reading. | +------------------------------+---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_pa_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from usami) author: John Snow Labs name: distilbert_qa_usami_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `usami`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_usami_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772969630.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_usami_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772969630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_usami_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_usami_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_usami_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/usami/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from anu24) author: John Snow Labs name: distilbert_qa_anu24_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `anu24`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_anu24_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769853124.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_anu24_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769853124.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anu24_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anu24_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_anu24_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anu24/distilbert-base-uncased-finetuned-squad --- layout: model title: Extract Anatomical Entities from Oncology Texts author: John Snow Labs name: ner_oncology_anatomy_general_wip date: 2022-09-30 tags: [licensed, clinical, oncology, en, ner, anatomy] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts anatomical entities using an unspecific label. Definitions of Predicted Entities: - `Anatomical_Site`: Relevant anatomical terms mentioned in text. - `Direction`: Directional and laterality terms, such as "left", "right", "bilateral", "upper" and "lower". ## Predicted Entities `Anatomical_Site`, `Direction` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_wip_en_4.0.0_3.0_1664562237279.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_wip_en_4.0.0_3.0_1664562237279.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:--------|:----------------| | left | Direction | | breast | Anatomical_Site | | lungs | Anatomical_Site | | liver | Anatomical_Site | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_anatomy_general_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|843.0 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Anatomical_Site 2377.0 649.0 353.0 2730.0 0.79 0.87 0.83 Direction 668.0 219.0 66.0 734.0 0.75 0.91 0.82 macro_avg 3045.0 868.0 419.0 3464.0 0.77 0.89 0.83 micro_avg NaN NaN NaN NaN 0.78 0.88 0.83 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from Zamachi) author: John Snow Labs name: roberta_qa_for_question_answering date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-for-question-answering` is a English model originally trained by `Zamachi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_for_question_answering_en_4.3.0_3.0_1674220787682.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_for_question_answering_en_4.3.0_3.0_1674220787682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_for_question_answering","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_for_question_answering","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_for_question_answering| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Zamachi/roberta-for-question-answering --- layout: model title: English image_classifier_vit_animal_classifier ViTForImageClassification from ritheshSree author: John Snow Labs name: image_classifier_vit_animal_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_animal_classifier` is a English model originally trained by ritheshSree. ## Predicted Entities `cat`, `dog`, `snake`, `tiger` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animal_classifier_en_4.1.0_3.0_1660170154919.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animal_classifier_en_4.1.0_3.0_1660170154919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_animal_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_animal_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_animal_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Detect Anatomical Structures (Single Entity - biobert) author: John Snow Labs name: ner_anatomy_coarse_biobert date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description An NER model to extract all types of anatomical references in text using "biobert_pubmed_base_cased" embeddings. It is a single entity model and generalizes all anatomical references to a single entity. ## Predicted Entities `Anatomy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ANATOMY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_en_3.0.0_3.0_1617209714335.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_en_3.0.0_3.0_1617209714335.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_anatomy_coarse_biobert", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["content in the lung tissue"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_anatomy_coarse_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner, ner_converter)) val data = Seq("""content in the lung tissue""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy.coarse_biobert").predict("""content in the lung tissue""") ```

## Results ```bash | | ner_chunk | entity | |---:|:------------------|:----------| | 0 | lung tissue | Anatomy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_coarse_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on a custom dataset using 'biobert_pubmed_base_cased'. ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | B-Anatomy | 2499 | 155 | 162 | 0.941598 | 0.939121 | 0.940357 | | 1 | I-Anatomy | 1695 | 116 | 158 | 0.935947 | 0.914733 | 0.925218 | | 2 | Macro-average | 4194 | 271 | 320 | 0.938772 | 0.926927 | 0.932812 | | 3 | Micro-average | 4194 | 271 | 320 | 0.939306 | 0.929109 | 0.93418 | ``` --- layout: model title: MeSH to UMLS Code Mapping author: John Snow Labs name: mesh_umls_mapping date: 2021-05-04 tags: [mesh, umls, en, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.0.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps MeSH codes to UMLS codes without using any text data. You’ll just feed white space-delimited MeSH codes and it will return the corresponding UMLS codes as a list. If there is no mapping, the original code is returned with no mapping. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapping_en_3.0.2_3.0_1620134296251.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapping_en_3.0.2_3.0_1620134296251.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("mesh_umls_mapping","en","clinical/models") pipeline.annotate("C028491 D019326 C579867") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("mesh_umls_mapping","en","clinical/models") val result = pipeline.annotate("C028491 D019326 C579867") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.mesh.umls").predict("""C028491 D019326 C579867""") ```

## Results ```bash {'mesh': ['C028491', 'D019326', 'C579867'], 'umls': ['C0970275', 'C0886627', 'C3696376']} Note: | MeSH | Details | | ---------- | ----------------------------:| | C028491 | 1,3-butylene glycol | | D019326 | 17-alpha-Hydroxyprogesterone | | C579867 | 3-Methylglutaconic Aciduria | | UMLS | Details | | ---------- | ---------------------------:| | C0970275 | 1,3-butylene glycol | | C0886627 | 17-hydroxyprogesterone | | C3696376 | 3-methylglutaconic aciduria | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|mesh_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.0.2+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_nl2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-nl2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl2_en_4.3.0_3.0_1675113785226.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl2_en_4.3.0_3.0_1675113785226.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_nl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_nl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_nl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|158.2 MB| ## References - https://huggingface.co/google/t5-efficient-base-nl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_deletion_squad_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-deletion-squad-10` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_deletion_squad_10_en_4.0.0_3.0_1655733844546.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_deletion_squad_10_en_4.0.0_3.0_1655733844546.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_deletion_squad_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_deletion_squad_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_deletion_10.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_deletion_squad_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-deletion-squad-10 --- layout: model title: Social Determinants of Health author: John Snow Labs name: ner_sdoh_wip date: 2023-02-11 tags: [licensed, clinical, en, social_determinants, ner, public_health, sdoh] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true recommended: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts terminology related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Other_SDoH_Keywords`, `Education`, `Population_Group`, `Quality_Of_Life`, `Housing`, `Substance_Frequency`, `Smoking`, `Eating_Disorder`, `Obesity`, `Healthcare_Institution`, `Financial_Status`, `Age`, `Chidhood_Event`, `Exercise`, `Communicable_Disease`, `Hypertension`, `Other_Disease`, `Violence_Or_Abuse`, `Spiritual_Beliefs`, `Employment`, `Social_Exclusion`, `Access_To_Care`, `Marital_Status`, `Diet`, `Social_Support`, `Disability`, `Mental_Health`, `Alcohol`, `Insurance_Status`, `Substance_Quantity`, `Hyperlipidemia`, `Family_Member`, `Legal_Issues`, `Race_Ethnicity`, `Gender`, `Geographic_Entity`, `Sexual_Orientation`, `Transportation`, `Sexual_Activity`, `Language`, `Substance_Use` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_NER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_wip_en_4.2.8_3.0_1676135569606.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_wip_en_4.2.8_3.0_1676135569606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = [["Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week."]] data = spark.createDataFrame(sample_texts).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("He continues to smoke one pack of cigarettes daily, as he has for the past 28 years.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +------------------+-----+---+-------------------+ |chunk |begin|end|ner_label | +------------------+-----+---+-------------------+ |55 years old |11 |22 |Age | |divorced |25 |32 |Marital_Status | |Mexcian American |34 |49 |Race_Ethnicity | |woman |51 |55 |Gender | |financial problems|62 |79 |Financial_Status | |She |82 |84 |Gender | |spanish |93 |99 |Language | |She |102 |104|Gender | |apartment |118 |126|Housing | |She |129 |131|Gender | |diabetes |158 |165|Other_Disease | |cleaning assistant|307 |324|Employment | |health insurance |354 |369|Insurance_Status | |She |391 |393|Gender | |son |401 |403|Family_Member | |student |405 |411|Education | |college |416 |422|Education | |depression |454 |463|Mental_Health | |She |466 |468|Gender | |she |479 |481|Gender | |rehab |489 |493|Access_To_Care | |her |514 |516|Gender | |catholic faith |518 |531|Spiritual_Beliefs | |support |547 |553|Social_Support | |She |565 |567|Gender | |etoh abuse |589 |598|Alcohol | |her |614 |616|Gender | |teens |618 |622|Age | |She |625 |627|Gender | |she |637 |639|Gender | |drinker |658 |664|Alcohol | |drinking beer |694 |706|Alcohol | |daily |708 |712|Substance_Frequency| |She |715 |717|Gender | |smokes |719 |724|Smoking | |a pack |726 |731|Substance_Quantity | |cigarettes |736 |745|Smoking | |a day |747 |751|Substance_Frequency| |She |754 |756|Gender | |DUI |762 |764|Legal_Issues | +------------------+-----+---+-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_wip| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.5 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Other_SDoH_Keywords 317.0 84.0 112.0 429.0 0.790524 0.738928 0.763855 Education 116.0 38.0 31.0 147.0 0.753247 0.789116 0.770764 Population_Group 27.0 4.0 11.0 38.0 0.870968 0.710526 0.782609 Quality_Of_Life 146.0 37.0 57.0 203.0 0.797814 0.719212 0.756477 Housing 809.0 91.0 118.0 927.0 0.898889 0.872708 0.885605 Substance_Frequency 74.0 19.0 44.0 118.0 0.795699 0.627119 0.701422 Smoking 136.0 4.0 2.0 138.0 0.971429 0.985507 0.978417 Eating_Disorder 40.0 2.0 0.0 40.0 0.952381 1.000000 0.975610 Obesity 16.0 1.0 5.0 21.0 0.941176 0.761905 0.842105 Healthcare_Institution 117.0 36.0 57.0 174.0 0.764706 0.672414 0.715596 Financial_Status 222.0 47.0 128.0 350.0 0.825279 0.634286 0.717286 Age 1328.0 109.0 48.0 1376.0 0.924148 0.965116 0.944188 Chidhood_Event 30.0 0.0 24.0 54.0 1.000000 0.555556 0.714286 Exercise 52.0 17.0 31.0 83.0 0.753623 0.626506 0.684211 Communicable_Disease 61.0 5.0 10.0 71.0 0.924242 0.859155 0.890511 Hypertension 45.0 1.0 12.0 57.0 0.978261 0.789474 0.873786 Other_Disease 1065.0 229.0 119.0 1184.0 0.823029 0.899493 0.859564 Violence_Or_Abuse 98.0 26.0 53.0 151.0 0.790323 0.649007 0.712727 Spiritual_Beliefs 94.0 9.0 21.0 115.0 0.912621 0.817391 0.862385 Employment 3797.0 272.0 288.0 4085.0 0.933153 0.929498 0.931322 Social_Exclusion 38.0 6.0 14.0 52.0 0.863636 0.730769 0.791667 Access_To_Care 810.0 95.0 160.0 970.0 0.895028 0.835052 0.864000 Marital_Status 177.0 4.0 9.0 186.0 0.977901 0.951613 0.964578 Diet 110.0 34.0 30.0 140.0 0.763889 0.785714 0.774648 Social_Support 1243.0 197.0 99.0 1342.0 0.863194 0.926230 0.893602 Disability 94.0 4.0 9.0 103.0 0.959184 0.912621 0.935323 Mental_Health 817.0 99.0 216.0 1033.0 0.891921 0.790900 0.838379 Alcohol 592.0 32.0 28.0 620.0 0.948718 0.954839 0.951768 Insurance_Status 145.0 23.0 32.0 177.0 0.863095 0.819209 0.840580 Substance_Quantity 107.0 42.0 39.0 146.0 0.718121 0.732877 0.725424 Hyperlipidemia 14.0 1.0 2.0 16.0 0.933333 0.875000 0.903226 Family_Member 4255.0 110.0 73.0 4328.0 0.974800 0.983133 0.978949 Legal_Issues 71.0 13.0 20.0 91.0 0.845238 0.780220 0.811429 Race_Ethnicity 81.0 9.0 7.0 88.0 0.900000 0.920455 0.910112 Gender 9698.0 183.0 193.0 9891.0 0.981480 0.980487 0.980983 Geographic_Entity 189.0 18.0 22.0 211.0 0.913043 0.895735 0.904306 Sexual_Orientation 21.0 0.0 3.0 24.0 1.000000 0.875000 0.933333 Transportation 27.0 2.0 27.0 54.0 0.931034 0.500000 0.650602 Sexual_Activity 56.0 4.0 24.0 80.0 0.933333 0.700000 0.800000 Language 35.0 6.0 2.0 37.0 0.853659 0.945946 0.897436 Substance_Use 400.0 40.0 24.0 424.0 0.909091 0.943396 0.925926 ``` --- layout: model title: Pipeline to Detect Anatomical Structures (Single Entity - biobert) author: John Snow Labs name: ner_anatomy_coarse_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_anatomy_coarse_biobert](https://nlp.johnsnowlabs.com/2021/03/31/ner_anatomy_coarse_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_pipeline_en_3.4.1_3.0_1647873505075.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_biobert_pipeline_en_3.4.1_3.0_1647873505075.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_anatomy_coarse_biobert_pipeline", "en", "clinical/models") pipeline.annotate("content in the lung tissue") ``` ```scala val pipeline = new PretrainedPipeline("ner_anatomy_coarse_biobert_pipeline", "en", "clinical/models") pipeline.annotate("content in the lung tissue") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy_coarse_biobert.pipeline").predict("""content in the lung tissue""") ```

## Results ```bash | | ner_chunk | entity | |---:|:------------------|:----------| | 0 | lung tissue | Anatomy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_coarse_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Chinese Word Segmentation author: John Snow Labs name: wordseg_large date: 2021-01-03 task: Word Segmentation language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, word_segmentation, zh, cn] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know _a priori_ which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. In this model, we created a curated large data set obtained from Chinese Treebank, Weibo, and SIGHAM 2005 data sets, and trained the neural network model as described in a research paper (Xue, Nianwen. "Chinese word segmentation as character tagging." International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_large_zh_2.7.0_2.4_1609681406666.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_large_zh_2.7.0_2.4_1609681406666.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.load("WORDSEG_LARGE_CN")\ .setInputCols("document")\ .setOutputCol("token")\ pipeline = Pipeline(stages=[document_assembler, word_segmenter]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['然而，这样的处理也衍生了一些问题。']], ["text"]) result = ws_model.transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_large", "zh") .setInputCols("document") .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("然而，这样的处理也衍生了一些问题。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，这样的处理也衍生了一些问题。"""] token_df = nlu.load('zh.segment_words.large').predict(text, output_level='token') token_df ```

## Results ```bash +----------------------------------+--------------------------------------------------------+ |text |result | +----------------------------------+--------------------------------------------------------+ |然而，这样的处理也衍生了一些问题。|[然而, ，, 这样, 的, 处理, 也, 衍生, 了, 一些, 问题, 。]| +----------------------------------+--------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_large| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|zh| ## Data Source cn_wordseg_large_train.chartag ## Benchmarking ```bash | Model | precision | recall | f1-score | |---------------|--------------|--------------|--------------| | WORSEG_CTB | 0,6453 | 0,6341 | 0,6397 | | WORDSEG_WEIBO | 0,5454 | 0,5655 | 0,5553 | | WORDSEG_MSRA | 0,5984 | 0,6088 | 0,6035 | | WORDSEG_PKU | 0,6094 | 0,6321 | 0,6206 | | WORDSEG_LARGE | 0,6326 | 0,6269 | 0,6297 | ``` --- layout: model title: Mapping ICD10CM Codes with Their Corresponding UMLS Codes author: John Snow Labs name: icd10cm_umls_mapper date: 2022-06-26 tags: [icd10cm, umls, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps ICD10CM codes to corresponding UMLS codes under the Unified Medical Language System (UMLS). ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapper_en_3.5.3_3.0_1656278690210.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapper_en_3.5.3_3.0_1656278690210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") icd10cm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_icd10cm","en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("icd10cm_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel\ .pretrained("icd10cm_umls_mapper", "en", "clinical/models")\ .setInputCols(["icd10cm_code"])\ .setOutputCol("umls_mappings")\ .setRels(["umls_code"]) pipeline = Pipeline(stages = [ documentAssembler, sbert_embedder, icd10cm_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline= LightPipeline(model) result = light_pipeline.fullAnnotate("Neonatal skin infection") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val icd10cm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_icd10cm", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel .pretrained("icd10cm_umls_mapper", "en", "clinical/models") .setInputCols(Array("rxnorm_code")) .setOutputCol("umls_mappings") .setRels(Array("umls_code")) val pipeline = new Pipeline(stages = Array( documentAssembler, sbert_embedder, icd10cm_resolver, chunkerMapper )) val data = Seq("Neonatal skin infection").toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.icd10cm_to_umls").predict("""Neonatal skin infection""") ```

## Results ```bash | | ner_chunk | icd10cm_code | umls_mappings | |---:|:------------------------|:---------------|:----------------| | 0 | Neonatal skin infection | P394 | C0456111 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_umls_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[icd10cm_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|942.9 KB| --- layout: model title: English Named Entity Recognition (from surrey-nlp) author: John Snow Labs name: roberta_ner_roberta_large_finetuned_abbr date: 2022-05-03 tags: [roberta, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-large-finetuned-abbr` is a English model orginally trained by `surrey-nlp`. ## Predicted Entities `LF`, `AC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_large_finetuned_abbr_en_3.4.2_3.0_1651594192589.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_large_finetuned_abbr_en_3.4.2_3.0_1651594192589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_roberta_large_finetuned_abbr| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/surrey-nlp/roberta-large-finetuned-abbr - https://paperswithcode.com/sota?task=Token+Classification&dataset=surrey-nlp%2FPLOD-unfiltered --- layout: model title: English Bert Embeddings (from alexanderfalk) author: John Snow Labs name: bert_embeddings_danbert_small_cased date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `danbert-small-cased` is a English model orginally trained by `alexanderfalk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_danbert_small_cased_en_3.4.2_3.0_1649672086620.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_danbert_small_cased_en_3.4.2_3.0_1649672086620.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_danbert_small_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|313.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/alexanderfalk/danbert-small-cased --- layout: model title: Smaller BERT Embeddings (L-4_H-128_A-2) author: John Snow Labs name: small_bert_L4_128 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L4_128_en_2.6.0_2.4_1598344330158.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L4_128_en_2.6.0_2.4_1598344330158.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L4_128", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L4_128", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L4_128').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L4_128_embeddings I [0.5109787583351135, 1.6565966606140137, 2.695.... love [1.0555483102798462, 1.8791943788528442, 1.285... NLP [-0.23064681887626648, 0.939659833908081, 1.77... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L4_128| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|128| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1](https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1) --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_kv128 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-kv128` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_kv128_en_4.3.0_3.0_1675112746492.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_kv128_en_4.3.0_3.0_1675112746492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_kv128","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_kv128","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_kv128| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|637.5 MB| ## References - https://huggingface.co/google/t5-efficient-base-kv128 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Firat) author: John Snow Labs name: distilbert_qa_firat_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Firat`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_firat_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768588053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_firat_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768588053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_firat_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_firat_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_firat_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Firat/distilbert-base-uncased-finetuned-squad --- layout: model title: German Bert Embeddings (Base, Cased) author: John Snow Labs name: bert_embeddings_gbert_base date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `gbert-base` is a German model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_gbert_base_de_3.4.2_3.0_1649675902802.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_gbert_base_de_3.4.2_3.0_1649675902802.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_gbert_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gbert-base - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: XLNet Embeddings (Base) author: John Snow Labs name: xlnet_base_cased date: 2020-04-28 task: Embeddings language: en nav_key: models edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: XlnetEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description XLNet is a new unsupervised language representation learning method based on a novel generalized permutation language modeling objective. Additionally, XLNet employs Transformer-XL as the backbone model, exhibiting excellent performance for language tasks involving long context. Overall, XLNet achieves state-of-the-art (SOTA) results on various downstream language tasks including question answering, natural language inference, sentiment analysis, and document ranking. The details are described in the paper "[XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlnet_base_cased_en_2.5.0_2.4_1588074114942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlnet_base_cased_en_2.5.0_2.4_1588074114942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = XlnetEmbeddings.pretrained("xlnet_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = XlnetEmbeddings.pretrained("xlnet_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.xlnet_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_xlnet_base_cased_embeddings I [0.0027268705889582634, -3.5811028480529785, 0... love [-4.020033836364746, -2.2760159969329834, 0.88... NLP [-0.2549888491630554, -2.2768502235412598, 1.1... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlnet_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/zihangdai/xlnet](https://github.com/zihangdai/xlnet) --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from Ninh) author: John Snow Labs name: xlmroberta_ner_ninh_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `Ninh`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_ninh_base_finetuned_panx_de_4.1.0_3.0_1660430037495.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_ninh_base_finetuned_panx_de_4.1.0_3.0_1660430037495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_ninh_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_ninh_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_ninh_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Ninh/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Pipeline to Detect PHI in Text author: John Snow Labs name: ner_deid_sd_large_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_sd_large](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_sd_large_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_pipeline_en_3.4.1_3.0_1647870104226.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_pipeline_en_3.4.1_3.0_1647870104226.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_sd_large_pipeline", "en", "clinical/models") pipeline.annotate("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_sd_large_pipeline", "en", "clinical/models") pipeline.annotate("A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.med_ner_large.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash +-----------------------------+--------+ |chunks |entities| +-----------------------------+--------+ |2093-01-13 |DATE | |David Hale |NAME | |Hendrickson, Ora |NAME | |7194334 |ID | |01/13/93 |DATE | |Oliveira |NAME | |1-11-2000 |DATE | |Cocke County Baptist Hospital|LOCATION| |0295 Keats Street |LOCATION| |786-5227 |CONTACT | |Brothers Coal-Mine |LOCATION| +-----------------------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_sd_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Bangla BertForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: bert_embeddings_indic_transformers date: 2022-12-02 tags: [bn, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: bn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-bn-bert` is a Bangla model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_bn_4.2.4_3.0_1670022315370.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_bn_4.2.4_3.0_1670022315370.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","bn") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","bn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ |relation|entity1|entity1_begin|entity1_end| chunk1|entity2|entity2_begin|entity2_end| chunk2|confidence| +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ | 1| TEST| 24| 25| CT| DATE| 30| 36| 1/12/95|0.99997973| | 1| TEST| 24| 25| CT|PROBLEM| 44| 83|progressive memor...| 0.9998983| | 1| TEST| 24| 25| CT| DATE| 91| 97| 8/11/94| 0.9997316| | 1| DATE| 30| 36| 1/12/95|PROBLEM| 44| 83|progressive memor...| 0.9998915| | 1| DATE| 30| 36| 1/12/95| DATE| 91| 97| 8/11/94| 0.9997931| | 1|PROBLEM| 44| 83|progressive memor...| DATE| 91| 97| 8/11/94| 0.9998667| +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_date_clinical_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References Trained on an internal dataset. ## Benchmarking ```bash label Recall Precision F1 Support 0 0.738 0.729 0.734 84 1 0.945 0.947 0.946 416 Avg. 0.841 0.838 0.840 - ``` --- layout: model title: Turkish BertForQuestionAnswering Base Cased model (from husnu) author: John Snow Labs name: bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3 date: 2022-07-07 tags: [tr, open_source, bert, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-turkish-128k-cased-finetuned_lr-2e-05_epochs-3` is a Turkish model originally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3_tr_4.0.0_3.0_1657183554228.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3_tr_4.0.0_3.0_1657183554228.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_turkish_128k_cased_finetuned_lr_2e_05_epochs_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|689.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/husnu/bert-base-turkish-128k-cased-finetuned_lr-2e-05_epochs-3 --- layout: model title: Fast Neural Machine Translation Model from English to Setswana author: John Snow Labs name: opus_mt_en_tn date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tn, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tn_xx_2.7.0_2.4_1609167140357.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tn_xx_2.7.0_2.4_1609167140357.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tn| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Chinese NER Model author: John Snow Labs name: bert_token_classifier_chinese_ner date: 2021-12-07 tags: [chinese, token_classifier, bert, zh, open_source] task: Named Entity Recognition language: zh edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` and it's been fine-tuned for traditional Chinese language, leveraging `Bert` embeddings and `BertForTokenClassification` for NER purposes. ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_chinese_ner_zh_3.3.2_2.4_1638881767667.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_chinese_ner_zh_3.3.2_2.4_1638881767667.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+---------+ |chunk |ner_label| +-----------------+---------+ |莎拉 |PERSON | |1999 年 11 月 2 |DATE | |斯图加特 |GPE | |梅赛德斯-奔驰公司 |ORG | +-----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_chinese_ner| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|zh| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/ckiplab/bert-base-chinese-ner](https://huggingface.co/ckiplab/bert-base-chinese-ner) ## Benchmarking ```bash label score f1 0.8118 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC2GM_Gene_Imbalancedscibert_scivocab_cased date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC2GM-Gene_Imbalancedscibert_scivocab_cased` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `GENE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC2GM_Gene_Imbalancedscibert_scivocab_cased_en_4.0.0_3.0_1657108037612.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC2GM_Gene_Imbalancedscibert_scivocab_cased_en_4.0.0_3.0_1657108037612.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC2GM_Gene_Imbalancedscibert_scivocab_cased| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC2GM-Gene_Imbalancedscibert_scivocab_cased --- layout: model title: English DistilBertForQuestionAnswering model (from charlieoneill) author: John Snow Labs name: distilbert_qa_base_uncased_gradient_clinic date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-gradient-clinic` is a English model originally trained by `charlieoneill`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_gradient_clinic_en_4.0.0_3.0_1654727035184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_gradient_clinic_en_4.0.0_3.0_1654727035184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_gradient_clinic","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_gradient_clinic","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_uncased.by_charlieoneill").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_gradient_clinic| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/charlieoneill/distilbert-base-uncased-gradient-clinic --- layout: model title: Thai Word Segmentation author: John Snow Labs name: wordseg_best date: 2021-01-13 task: Word Segmentation language: th edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [th, word_segmentation, open_source] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Thai text. Thai text is written without white space between the words, and a computer-based application cannot know _a priori_ which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. References: - Xue, Nianwen. "Chinese word segmentation as character tagging." International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_best_th_2.7.0_2.4_1610543628078.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_best_th_2.7.0_2.4_1610543628078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline as a substitute of the Tokenizer stage.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.pretrained('wordseg_best', 'th')\ .setInputCols("document")\ .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler, word_segmenter]) example = spark.createDataFrame([['จวนจะถึงร้านที่คุณจองโต๊ะไว้แล้วจ้ะ']], ["text"]) result = pipeline.fit(example ).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_best", "th") .setInputCols("document") .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("จวนจะถึงร้านที่คุณจองโต๊ะไว้แล้วจ้ะ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Mona Lisa เป็นภาพวาดสีน้ำมันในศตวรรษที่ 16 ที่สร้างโดย Leonardo จัดขึ้นที่พิพิธภัณฑ์ลูฟร์ในปารีส"""] token_df = nlu.load('th.segment_words').predict(text) token_df ```

## Results ```bash +-----------------------------------+---------------------------------------------------------+ |text |result | +-----------------------------------+---------------------------------------------------------+ |จวนจะถึงร้านที่คุณจองโต๊ะไว้แล้วจ้ะ|[จวน, จะ, ถึง, ร้าน, ที่, คุณ, จอง, โต๊ะ, ไว้, แล้ว, จ้ะ]| +-----------------------------------+---------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_best| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|th| ## Data Source The model was trained on the [BEST](http://thailang.nectec.or.th/best) corpus from the National Electronics and Computer Technology Center (NECTEC). References: > - Krit Kosawat, Monthika Boriboon, Patcharika Chootrakool, Ananlada Chotimongkol, Supon Klaithin, Sarawoot Kongyoung, Kanyanut Kriengket, Sitthaa Phaholphinyo, Sumonmas Purodakananda, Tipraporn Thanakulwarapas, and Chai Wutiwiwatchai, "BEST 2009: Thai word segmentation software contest," in Proc. 8th Int. Symp. Natural Language Process. (SNLP), Bangkok, Thailand, Oct.20-22, 2009, pp.83-88. > - Monthika Boriboon, Kanyanut Kriengket, Patcharika Chootrakool, Sitthaa Phaholphinyo, Sumonmas Purodakananda, Tipraporn Thanakulwarapas, and Krit Kosawat, "BEST corpus development and analysis," in Proc. 2nd Int. Conf. Asian Language Process. (IALP), Singapore, Dec.7-9, 2009, pp.322-327. ## Benchmarking ```bash | Model | precision | recall | f1-score | |--------------|-----------|--------|----------| | WORDSEG_BEST | 0.4791 | 0.6245 | 0.5422 | ``` --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nl2 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nl2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl2_en_4.3.0_3.0_1675123819194.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl2_en_4.3.0_3.0_1675123819194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|39.0 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English asr_wav2vec2_base_100h_with_lm_by_saahith TFWav2Vec2ForCTC from saahith author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_with_lm_by_saahith` is a English model originally trained by saahith. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith_en_4.2.0_3.0_1664117830342.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith_en_4.2.0_3.0_1664117830342.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_with_lm_by_saahith| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Adverse Drug Events (healthcare) author: John Snow Labs name: ner_ade_healthcare date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse drug events in tweets, reviews, and medical text using pretrained NER model. ## Predicted Entities `DRUG`, `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_ade_healthcare_en_3.0.0_3.0_1617260836627.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_ade_healthcare_en_3.0.0_3.0_1617260836627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_ade_healthcare", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_ade_healthcare", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.ade.ade_healthcare").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ade_healthcare| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash +------+------+------+------+-------+---------+------+------+ |entity| tp| fp| fn| total|precision|recall| f1| +------+------+------+------+-------+---------+------+------+ | DRUG|9649.0| 884.0|9772.0|19421.0| 0.9161|0.4968|0.6443| | ADE|5909.0|9508.0|1987.0| 7896.0| 0.3833|0.7484|0.5069| +------+------+------+------+-------+---------+------+------+ +------------------+ | macro| +------------------+ |0.5755909944827655| +------------------+ +------------------+ | micro| +------------------+ |0.6045600310939989| +------------------+ ``` --- layout: model title: Pipeline to Resolve CVX Codes author: John Snow Labs name: cvx_resolver_pipeline date: 2023-03-30 tags: [en, licensed, clinical, resolver, chunk_mapping, cvx, pipeline] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities with their corresponding CVX codes. You’ll just feed your text and it will return the corresponding CVX codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/cvx_resolver_pipeline_en_4.3.2_3.2_1680178011294.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/cvx_resolver_pipeline_en_4.3.2_3.2_1680178011294.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline resolver_pipeline = PretrainedPipeline("cvx_resolver_pipeline", "en", "clinical/models") text= "The patient has a history of influenza vaccine, tetanus and DTaP" result = resolver_pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val resolver_pipeline = new PretrainedPipeline("cvx_resolver_pipeline", "en", "clinical/models") val result = resolver_pipeline.fullAnnotate("The patient has a history of influenza vaccine, tetanus and DTaP") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.cvx_pipeline").predict("""The patient has a history of influenza vaccine, tetanus and DTaP""") ```

## Results ```bash +-----------------+---------+--------+ |chunk |ner_chunk|cvx_code| +-----------------+---------+--------+ |influenza vaccine|Vaccine |160 | |tetanus |Vaccine |35 | |DTaP |Vaccine |20 | +-----------------+---------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|cvx_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|2.1 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: Detect Assertion Status from Oncology Entities author: John Snow Labs name: assertion_oncology_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of entities related to oncology (including diagnoses, therapies and tests). ## Predicted Entities `Absent`, `Family`, `Hypothetical`, `Past`, `Possible`, `Present` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_wip_en_4.1.0_3.0_1664641275549.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_wip_en_4.1.0_3.0_1664641275549.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------------|:---------------|:------------| | breast cancer | Cancer_Dx | Possible | | cancers | Cancer_Dx | Family | | biopsy | Pathology_Test | Past | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Absent 0.81 0.77 0.79 264.0 Family 0.78 0.82 0.80 34.0 Hypothetical 0.67 0.61 0.64 182.0 Past 0.91 0.93 0.92 1583.0 Possible 0.59 0.59 0.59 51.0 Present 0.89 0.89 0.89 1645.0 macro-avg 0.77 0.77 0.77 3759.0 weighted-avg 0.88 0.88 0.88 3759.0 ``` --- layout: model title: Finnish RobertaForQuestionAnswering (from cgou) author: John Snow Labs name: roberta_qa_fin_RoBERTa_v1_finetuned_squad date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: fi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fin_RoBERTa-v1-finetuned-squad` is a Finnish model originally trained by `cgou`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fin_RoBERTa_v1_finetuned_squad_fi_4.0.0_3.0_1655728569389.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fin_RoBERTa_v1_finetuned_squad_fi_4.0.0_3.0_1655728569389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_fin_RoBERTa_v1_finetuned_squad","fi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_fin_RoBERTa_v1_finetuned_squad","fi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fi.answer_question.squad.roberta.by_cgou").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fin_RoBERTa_v1_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fi| |Size:|248.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/cgou/fin_RoBERTa-v1-finetuned-squad --- layout: model title: Mapping Drugs from the KEGG Database to Their Efficacies, Molecular Weights and Corresponding Codes from Other Databases author: John Snow Labs name: kegg_drug_mapper date: 2022-11-21 tags: [drug, efficacy, molecular_weight, cas, pubchem, chebi, ligandbox, nikkaji, pdbcct, chunk_mapper, clinical, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps drugs with their corresponding `efficacy`, `molecular_weight` as well as `CAS`, `PubChem`, `ChEBI`, `LigandBox`, `NIKKAJI`, `PDB-CCD` codes. This model was trained with the data from the KEGG database. ## Predicted Entities `efficacy`, `molecular_weight`, `CAS`, `PubChem`, `ChEBI`, `LigandBox`, `NIKKAJI`, `PDB-CCD` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/kegg_drug_mapper_en_4.2.2_3.0_1669069910375.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/kegg_drug_mapper_en_4.2.2_3.0_1669069910375.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained("ner_posology", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ chunkerMapper = ChunkMapperModel.pretrained("kegg_drug_mapper", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["efficacy", "molecular_weight", "CAS", "PubChem", "ChEBI", "LigandBox", "NIKKAJI", "PDB-CCD"])\ pipeline = Pipeline().setStages([ document_assembler, sentence_detector, tokenizer, word_embeddings, ner, converter, chunkerMapper]) text= "She is given OxyContin, folic acid, levothyroxine, Norvasc, aspirin, Neurontin" data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_posology", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val chunkerMapper = ChunkMapperModel.pretrained("kegg_drug_mapper", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("mappings") .setRels(Array("efficacy", "molecular_weight", "CAS", "PubChem", "ChEBI", "LigandBox", "NIKKAJI", "PDB-CCD")) val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner, converter, chunkerMapper)) val text= "She is given OxyContin, folic acid, levothyroxine, Norvasc, aspirin, Neurontin" val data = Seq(text).toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.kegg_drug").predict("""She is given OxyContin, folic acid, levothyroxine, Norvasc, aspirin, Neurontin""") ```

## Results ```bash +-------------+--------------------------------------------------+----------------+----------+-----------+-------+---------+---------+-------+ | ner_chunk| efficacy|molecular_weight| CAS| PubChem| ChEBI|LigandBox| NIKKAJI|PDB-CCD| +-------------+--------------------------------------------------+----------------+----------+-----------+-------+---------+---------+-------+ | OxyContin| Analgesic (narcotic), Opioid receptor agonist| 351.8246| 124-90-3| 7847912.0| 7859.0| D00847|J281.239H| NONE| | folic acid|Anti-anemic, Hematopoietic, Supplement (folic a...| 441.3975| 59-30-3| 7847138.0|27470.0| D00070| J1.392G| FOL| |levothyroxine| Replenisher (thyroid hormone)| 776.87| 51-48-9|9.6024815E7|18332.0| D08125| J4.118A| T44| | Norvasc|Antihypertensive, Vasodilator, Calcium channel ...| 408.8759|88150-42-9|5.1091781E7| 2668.0| D07450| J33.383B| NONE| | aspirin|Analgesic, Anti-inflammatory, Antipyretic, Anti...| 180.1574| 50-78-2| 7847177.0|15365.0| D00109| J2.300K| AIN| | Neurontin| Anticonvulsant, Antiepileptic| 171.2368|60142-96-3| 7847398.0|42797.0| D00332| J39.388F| GBN| +-------------+--------------------------------------------------+----------------+----------+-----------+-------+---------+---------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|kegg_drug_mapper| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|1.0 MB| --- layout: model title: Translate English to Haitian Creole Pipeline author: John Snow Labs name: translate_en_ht date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ht, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ht` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ht_xx_2.7.0_2.4_1609688301365.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ht_xx_2.7.0_2.4_1609688301365.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ht", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ht", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ht').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ht| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German Bert Embeddings (Base, Cased, Old Vocabulary) author: John Snow Labs name: bert_embeddings_bert_base_german_cased_oldvocab date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-german-cased-oldvocab` is a German model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_cased_oldvocab_de_3.4.2_3.0_1649676274361.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_cased_oldvocab_de_3.4.2_3.0_1649676274361.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_german_cased_oldvocab| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|409.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/bert-base-german-cased-oldvocab - https://github.com/deepset-ai/FARM/issues/60 - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Persian BertForQuestionAnswering model (from SajjadAyoubi) author: John Snow Labs name: bert_qa_bert_base_fa_qa date: 2022-06-02 tags: [fa, open_source, question_answering, bert] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-fa-qa` is a Persian model orginally trained by `SajjadAyoubi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_fa_qa_fa_4.0.0_3.0_1654179918056.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_fa_qa_fa_4.0.0_3.0_1654179918056.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_fa_qa","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_fa_qa","fa") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fa.answer_question.bert.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_fa_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SajjadAyoubi/bert-base-fa-qa - https://colab.research.google.com/github/sajjjadayobi/PersianQA/blob/main/notebooks/HowToUse.ipynb --- layout: model title: Translate English to East Slavic languages Pipeline author: John Snow Labs name: translate_en_zle date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, zle, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `zle` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_zle_xx_2.7.0_2.4_1609691744462.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_zle_xx_2.7.0_2.4_1609691744462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_zle", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_zle", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.zle').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_zle| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Universal Sentence Encoder XLING Many author: John Snow Labs name: tfhub_use_xling_many date: 2020-12-08 task: Embeddings language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 deprecated: true tags: [embeddings, open_source, xx] supported: true annotator: UniversalSentenceEncoder article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The Universal Sentence Encoder Cross-lingual (XLING) module is an extension of the Universal Sentence Encoder that includes training on multiple tasks across languages. The multi-task training setup is based on the paper "Learning Cross-lingual Sentence Representations via a Multi-task Dual Encoder". This specific module is trained on English, French, German, Spanish, Italian, Chinese, Korean, and Japanese tasks, and optimized for greater-than-word length text, such as sentences, phrases or short paragraphs. It is trained on a variety of data sources and tasks, with the goal of learning text representations that are useful out-of-the-box for a number of applications. The input to the module is variable length text in any of the eight aforementioned languages and the output is a 512 dimensional vector. We note that one does not need to specify the language of the input, as the model was trained such that text across languages with similar meanings will have embeddings with high dot product scores. Note: This model only works on Linux and macOS operating systems and is not compatible with Windows due to the incompatibility of the SentencePiece library. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/tfhub_use_xling_many_xx_2.7.0_2.4_1607440840968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/tfhub_use_xling_many_xx_2.7.0_2.4_1607440840968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_xling_many", "xx") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP', 'Me encanta usar SparkNLP']], ["text"])) ``` ```scala val embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_xling_many", "xx") .setInputCols("document") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I love NLP", "Me encanta usar SparkNLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP", "Me encanta usar SparkNLP"] embeddings_df = nlu.load('xx.use.xling_many').predict(text, output_level='sentence') embeddings_df ```

## Results It gives a 512-dimensional vector of the sentences. ```bash xx_use_xling_many_embeddings sentence 0 [0.03621278703212738, 0.007045685313642025, -0... I love NLP 1 [-0.0060035050846636295, 0.028749311342835426,... Me encanta usar SparkNLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|tfhub_use_xling_many| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|xx| ## Data Source [https://tfhub.dev/google/universal-sentence-encoder-xling-many/1](https://tfhub.dev/google/universal-sentence-encoder-xling-many/1) --- layout: model title: Pipeline to Mapping MESH Codes with Their Corresponding UMLS Codes author: John Snow Labs name: mesh_umls_mapping date: 2022-06-27 tags: [mesh, umls, chunk_mapper, pipeline, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `mesh_umls_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapping_en_3.5.3_3.0_1656366727552.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapping_en_3.5.3_3.0_1656366727552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("mesh_umls_mapping", "en", "clinical/models") result = pipeline.fullAnnotate("C028491 D019326 C579867") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= new PretrainedPipeline("mesh_umls_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate("C028491 D019326 C579867") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.mesh.umls").predict("""C028491 D019326 C579867""") ```

## Results ```bash | | mesh_code | umls_code | |---:|:----------------------------|:-------------------------------| | 0 | C028491 | D019326 | C579867 | C0043904 | C0045010 | C3696376 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|mesh_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.8 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nh16 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nh16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nh16_en_4.3.0_3.0_1675123654269.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nh16_en_4.3.0_3.0_1675123654269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nh16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nh16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nh16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|64.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nh16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Extract relations between phenotypic abnormalities and diseases (ReDL) author: John Snow Labs name: redl_human_phenotype_gene_biobert date: 2021-07-24 tags: [relation_extraction, en, licensed, clinical] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 2.4 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations to fully understand the origin of some phenotypic abnormalities and their associated diseases. `1` : Entities are related, `0` : Entities are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_3.0.3_2.4_1627120647767.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_3.0.3_2.4_1627120647767.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|-----------:|:----------|----------------:|--------------:|:---------------------|:----------|----------------:|--------------:|:--------------------|-------------:| | 0 | 0 | HP | 10 | 29 | retinal degeneration | HP | 32 | 43 | hearing loss | 0.893809 | | 1 | 0 | HP | 10 | 29 | retinal degeneration | HP | 49 | 61 | renal failure | 0.958486 | | 2 | 1 | HP | 10 | 29 | retinal degeneration | HP | 162 | 180 | autosomal recessive | 0.65584 | | 3 | 0 | HP | 32 | 43 | hearing loss | HP | 64 | 76 | short stature | 0.707055 | | 4 | 1 | HP | 32 | 43 | hearing loss | GENE | 96 | 103 | SH3PXD2B | 0.640802 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_human_phenotype_gene_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on a silver standard corpus of human phenotype and gene annotations and their relations. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.922 0.908 0.915 129 1 0.831 0.855 0.843 71 Avg. 0.877 0.882 0.879 - ``` --- layout: model title: English image_classifier_vit_exper3_mesum5 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_exper3_mesum5 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_exper3_mesum5` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper3_mesum5_en_4.1.0_3.0_1660167974762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper3_mesum5_en_4.1.0_3.0_1660167974762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_exper3_mesum5", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_exper3_mesum5", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_exper3_mesum5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: Translate Bulgarian to English Pipeline author: John Snow Labs name: translate_bg_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bg, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bg` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bg_en_xx_2.7.0_2.4_1609691570462.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bg_en_xx_2.7.0_2.4_1609691570462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bg_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bg_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bg.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bg_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Financial SEC Filings Classifier author: John Snow Labs name: finclf_sec_filings date: 2022-12-01 tags: [en, finance, classification, sec, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model allows you to classify documents among a list of specific US Security Exchange Commission filings, as : `10-K`, `10-Q`, `8-K`, `S-8`, `3`, `4`, `Other` **IMPORTANT** : This model works with the first 512 tokens of a document, you don't need to run it in the whole document. ## Predicted Entities `10-K`, `10-Q`, `8-K`, `S-8`, `3`, `4`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_sec_filings_en_1.0.0_3.0_1669921534523.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_sec_filings_en_1.0.0_3.0_1669921534523.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = finance.ClassifierDLModel.pretrained("finclf_sec_filings", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier ]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[10-K]| |[8-K]| |[10-Q]| |[S-8]| |[3]| |[4]| |[other]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_sec_filings| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Scrapped filings from SEC ## Benchmarking ```bash label precision recall f1-score support 10-K 0.97 0.82 0.89 40 10-Q 0.94 0.94 0.94 35 3 0.80 0.95 0.87 41 4 0.94 0.76 0.84 42 8-K 0.81 0.94 0.87 32 S-8 0.91 0.93 0.92 44 other 0.98 0.98 0.98 41 accuracy - - 0.90 275 macro-avg 0.91 0.90 0.90 275 weighted-avg 0.91 0.90 0.90 275 ``` --- layout: model title: Embeddings Scielo 150 dims author: John Snow Labs name: embeddings_scielo_150d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-26 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_150d_es_2.5.0_2.4_1590467082526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_150d_es_2.5.0_2.4_1590467082526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_scielo_150d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_scielo_150d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.scielo.150d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_scielo_150d``. {:.model-param} ## Model Information {:.table-model} |---------------|------------------------| | Name: | embeddings_scielo_150d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | es | | Dimension: | 150.0 | {:.h2_title} ## Data Source Trained on Scielo Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: Pipeline to Detect Organism in Medical Text author: John Snow Labs name: bert_token_classifier_ner_species_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_species](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_species_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_species_pipeline_en_4.3.0_3.2_1679301125473.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_species_pipeline_en_4.3.0_3.2_1679301125473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_species_pipeline", "en", "clinical/models") text = '''As determined by 16S rRNA gene sequence analysis, strain 6C (T) represents a distinct species belonging to the class Betaproteobacteria and is most closely related to Thiomonas intermedia DSM 18155 (T) and Thiomonas perometabolis DSM 18570 (T) .''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_species_pipeline", "en", "clinical/models") val text = "As determined by 16S rRNA gene sequence analysis, strain 6C (T) represents a distinct species belonging to the class Betaproteobacteria and is most closely related to Thiomonas intermedia DSM 18155 (T) and Thiomonas perometabolis DSM 18570 (T) ." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------|--------:|------:|:------------|-------------:| | 0 | 6C (T) | 57 | 62 | SPECIES | 0.998955 | | 1 | Betaproteobacteria | 117 | 134 | SPECIES | 0.99973 | | 2 | Thiomonas intermedia | 167 | 186 | SPECIES | 0.999822 | | 3 | DSM 18155 (T) | 188 | 200 | SPECIES | 0.997657 | | 4 | Thiomonas perometabolis | 206 | 228 | SPECIES | 0.999614 | | 5 | DSM 18570 (T) | 230 | 242 | SPECIES | 0.997146 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from ZYW) author: John Snow Labs name: distilbert_qa_zyw_model date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `en-de-vi-zh-es-model` is a Multilingual model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_zyw_model_xx_4.3.0_3.0_1672775050259.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_zyw_model_xx_4.3.0_3.0_1672775050259.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zyw_model","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zyw_model","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_zyw_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/en-de-vi-zh-es-model --- layout: model title: Extract Clinical Department Entities from Voice of the Patient Documents (embeddings_clinical_large) author: John Snow Labs name: ner_vop_clinical_dept_emb_clinical_large date: 2023-06-06 tags: [licensed, clinical, en, ner, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts medical devices and clinical department mentions terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `ClinicalDept`, `AdmissionDischarge`, `MedicalDevice` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_emb_clinical_large_en_4.4.3_3.0_1686074681308.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_emb_clinical_large_en_4.4.3_3.0_1686074681308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------------------|:--------------| | orthopedic department | ClinicalDept | | titanium plate | MedicalDevice | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_clinical_dept_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_large| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 ClinicalDept 297 35 29 326 0.89 0.91 0.90 AdmissionDischarge 25 0 9 34 1.00 0.74 0.85 MedicalDevice 256 64 76 332 0.80 0.77 0.79 macro_avg 578 99 114 692 0.90 0.81 0.85 micro_avg 578 99 114 692 0.85 0.83 0.84 ``` --- layout: model title: Legal Eu Finance Document Classifier (EURLEX) author: John Snow Labs name: legclf_eu_finance_bert date: 2023-03-06 tags: [en, legal, classification, clauses, eu_finance, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_eu_finance_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Eu_Finance or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Eu_Finance`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_eu_finance_bert_en_1.0.0_3.0_1678111884579.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_eu_finance_bert_en_1.0.0_3.0_1678111884579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_eu_finance_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Eu_Finance]| |[Other]| |[Other]| |[Eu_Finance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_eu_finance_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Eu_Finance 0.89 0.87 0.88 622 Other 0.85 0.87 0.86 529 accuracy - - 0.87 1151 macro-avg 0.87 0.87 0.87 1151 weighted-avg 0.87 0.87 0.87 1151 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_rbt4 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rbt4` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_zh_4.2.4_3.0_1670327086297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_zh_4.2.4_3.0_1670327086297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt4","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt4","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt4| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|171.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt4 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Detect Drug Chemicals author: John Snow Labs name: ner_drugs_large_en date: 2021-01-29 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 2.7.1 spark_version: 2.4 tags: [ner, en, licensed, clinical] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Pretrained named entity recognition deep learning model for Drugs. The model combines dosage, strength, form, and route into a single entity: Drug. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. {:.h2_title} ## Predicted Entities `DRUG` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugs_large_en_2.6.0_2.4_1603915964112.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugs_large_en_2.6.0_2.4_1603915964112.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPython.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") # Clinical word embeddings trained on PubMED dataset word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_drugs_large", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) data = spark.createDataFrame([["""The patient is a 40-year-old white male who presents with a chief complaint of 'chest pain'. The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. He has been advised Aspirin 81 milligrams QDay. Humulin N. insulin 50 units in a.m. HCTZ 50 mg QDay. Nitroglycerin 1/150 sublingually PRN chest pain."""]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") # Clinical word embeddings trained on PubMED dataset val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_drugs_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The patient is a 40-year-old white male who presents with a chief complaint of 'chest pain'. The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. He has been advised Aspirin 81 milligrams QDay. Humulin N. insulin 50 units in a.m. HCTZ 50 mg QDay. Nitroglycerin 1/150 sublingually PRN chest pain.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results The output is a dataframe with a sentence per row and a ``"ner"`` column containing all of the entity labels in the sentence, entity character indices, and other metadata. ```bash +--------------------------------+---------+ |chunk |ner_label| +--------------------------------+---------+ |Aspirin 81 milligrams |DRUG | |Humulin N |DRUG | |insulin 50 units |DRUG | |HCTZ 50 mg |DRUG | |Nitroglycerin 1/150 sublingually|DRUG | +--------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_large_en_2.6.0_2.4| |Type:|ner| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence,token, embeddings]| |Output Labels:|[ner]| |Language:|[en]| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on i2b2_med7 + FDA with 'embeddings_clinical'. https://www.i2b2.org/NLP/Medication {:.h2_title} ## Benchmarking Since this NER model is crafted from `ner_posology` but reduced to single entity, no benchmark is applicable. --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265908` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908_en_4.0.0_3.0_1655985559166.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908_en_4.0.0_3.0_1655985559166.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265908").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265908| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265908 --- layout: model title: English image_classifier_vit_pond_image_classification_7 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_pond_image_classification_7 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pond_image_classification_7` is a English model originally trained by SummerChiam. ## Predicted Entities `Normal`, `Boiling`, `Algae`, `NormalCement`, `NormalRain`, `BoilingNight`, `NormalNight` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_7_en_4.1.0_3.0_1660167146003.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_7_en_4.1.0_3.0_1660167146003.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_7", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_7", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_7| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Earning Calls Financial NER (Specific, md) author: John Snow Labs name: finner_earning_calls_specific_md date: 2022-12-15 tags: [en, finance, ner, licensed, earning, calls] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a `md` (medium) version of a financial model trained on Earning Calls transcripts to detect financial entities (NER model). This model is called `Specific` as it has more labels in comparison with the `Generic` version. Please note this model requires some tokenization configuration to extract the currency (see python snippet below). The currently available entities are: - AMOUNT: Numeric amounts, not percentages - ASSET: Current or Fixed Asset - ASSET_DECREASE: Decrease in the asset possession/exposure - ASSET_INCREASE: Increase in the asset possession/exposure - CF: Total cash flow - CFO: Cash flow from operating activity - CFO_INCREASE: Cash flow from operating activity increased - CONTRA_LIABILITY: Negative liability account that offsets the liability account (e.g. paying a debt) - COUNT: Number of items (not monetary, not percentages). - CURRENCY: The currency of the amount - DATE: Generic dates in context where either it's not a fiscal year or it can't be asserted as such given the context - EXPENSE: An expense or loss - EXPENSE_DECREASE: A piece of information saying there was an expense decrease in that fiscal year - EXPENSE_INCREASE: A piece of information saying there was an expense increase in that fiscal year - FCF: Free Cash Flow - FISCAL_YEAR: A date which expresses which month the fiscal exercise was closed for a specific year - INCOME: Any income that is reported - INCOME_INCREASE: Relative increase in income - KPI: Key Performance Indicator, a quantifiable measure of performance over time for a specific objective - KPI_DECREASE: Relative decrease in a KPI - KPI_INCREASE: Relative increase in a KPI - LIABILITY: Current or Long-Term Liability (not from stockholders) - LIABILITY_DECREASE: Relative decrease in liability - LIABILITY_INCREASE: Relative increase in liability - LOSS: Type of loss (e.g. gross, net) - ORG: Mention to a company/organization name - PERCENTAGE: : Numeric amounts which are percentages - PROFIT: Profit or also Revenue - PROFIT_DECLINE: A piece of information saying there was a profit / revenue decrease in that fiscal year - PROFIT_INCREASE: A piece of information saying there was a profit / revenue increase in that fiscal year - REVENUE: Revenue reported by company - REVENUE_DECLINE: Relative decrease in revenue when compared to other years - REVENUE_INCREASE: Relative increase in revenue when compared to other years - STOCKHOLDERS_EQUITY: Equity possessed by stockholders, not liability - TICKER: Trading symbol of the company ## Predicted Entities `AMOUNT`, `ASSET`, `ASSET_DECREASE`, `ASSET_INCREASE`, `CF`, `CFO`, `CFO_INCREASE`, `CF_INCREASE`, `CONTRA_LIABILITY`, `COUNT`, `CURRENCY`, `DATE`, `EXPENSE`, `EXPENSE_DECREASE`, `EXPENSE_INCREASE`, `FCF`, `FISCAL_YEAR`, `INCOME`, `INCOME_INCREASE`, `KPI`, `KPI_DECREASE`, `KPI_INCREASE`, `LIABILITY`, `LIABILITY_DECREASE`, `LIABILITY_INCREASE`, `LOSS`, `LOSS_DECREASE`, `ORG`, `PERCENTAGE`, `PROFIT`, `PROFIT_DECLINE`, `PROFIT_INCREASE`, `REVENUE`, `REVENUE_DECLINE`, `REVENUE_INCREASE`, `STOCKHOLDERS_EQUITY`, `TICKER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_specific_md_en_1.0.0_3.0_1671134641020.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_specific_md_en_1.0.0_3.0_1671134641020.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_earning_calls_specific_md", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Adjusted EPS was ahead of our expectations at $ 1.21 , and free cash flow is also ahead of our expectations despite a $ 1.5 billion additional tax payment we made related to the R&D amortization."""]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("text"), F.expr("cols['1']['entity']").alias("label")).show(200, truncate = False) ```

## Results ```bash +------------+----------+----------+ | token| ner_label|confidence| +------------+----------+----------+ | Adjusted| B-PROFIT| 0.6179| | EPS| I-PROFIT| 0.913| | was| O| 1.0| | ahead| O| 1.0| | of| O| 1.0| | our| O| 1.0| |expectations| O| 1.0| | at| O| 1.0| | $|B-CURRENCY| 1.0| | 1.21| B-AMOUNT| 1.0| | ,| O| 1.0| | and| O| 1.0| | free| B-FCF| 0.9992| | cash| I-FCF| 0.9945| | flow| I-FCF| 0.9988| | is| O| 1.0| | also| O| 1.0| | ahead| O| 1.0| | of| O| 1.0| | our| O| 1.0| |expectations| O| 1.0| | despite| O| 1.0| | a| O| 1.0| | $|B-CURRENCY| 1.0| | 1.5| B-AMOUNT| 1.0| | billion| I-AMOUNT| 1.0| | additional| O| 0.9945| | tax| O| 0.6131| | payment| O| 0.6613| | we| O| 1.0| | made| O| 1.0| | related| O| 1.0| | to| O| 1.0| | the| O| 1.0| | R&D| O| 0.9994| |amortization| O| 0.9989| | .| O| 1.0| +------------+----------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_earning_calls_specific_md| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on Earning Calls. ## Benchmarking ```bash label precision recall f1 support AMOUNT 99.303136 99.650350 99.476440 574 ASSET 55.172414 47.058824 50.793651 29 ASSET_INCREASE 100.000000 33.333333 50.000000 1 CF 46.153846 70.588235 55.813953 26 CFO 77.777778 100.000000 87.500000 9 CONTRA_LIABILITY 52.380952 56.410256 54.320988 42 COUNT 65.384615 77.272727 70.833333 26 CURRENCY 98.916968 99.636364 99.275362 554 DATE 86.982249 93.630573 90.184049 169 EXPENSE 67.187500 57.333333 61.870504 64 EXPENSE_DECREASE 100.000000 60.000000 75.000000 3 EXPENSE_INCREASE 40.000000 44.444444 42.105263 10 FCF 75.000000 75.000000 75.000000 20 INCOME 60.000000 40.000000 48.000000 10 KPI 41.666667 23.809524 30.303030 12 KPI_DECREASE 20.000000 10.000000 13.333333 5 KPI_INCREASE 44.444444 38.095238 41.025641 18 LIABILITY 38.461538 38.461538 38.461538 13 LIABILITY_DECREASE 50.000000 66.666667 57.142857 4 LOSS 50.000000 37.500000 42.857143 6 ORG 94.736842 90.000000 92.307692 19 PERCENTAGE 99.299475 99.648506 99.473684 571 PROFIT 78.014184 85.937500 81.784387 141 PROFIT_DECLINE 100.000000 36.363636 53.333333 4 PROFIT_INCREASE 78.947368 75.000000 76.923077 19 REVENUE 64.835165 71.951220 68.208092 91 REVENUE_DECLINE 53.571429 57.692308 55.555556 28 REVENUE_INCREASE 65.734266 75.200000 70.149254 143 STOCKHOLDERS_EQUITY 60.000000 37.500000 46.153846 5 TICKER 94.444444 94.444444 94.444444 18 accuracy - - 0.9571 19083 macro-avg 0.6660 0.5900 0.6070 19083 weighted-avg 0.9575 0.9571 0.9563 19083 ``` --- layout: model title: Legal Note Purchase Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_note_purchase_agreement date: 2022-11-24 tags: [en, legal, classification, agreement, note_purchase, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_note_purchase_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `note-purchase-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `note-purchase-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_note_purchase_agreement_en_1.0.0_3.0_1669292848323.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_note_purchase_agreement_en_1.0.0_3.0_1669292848323.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_holtin_finetuned_full_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_holtin_finetuned_full_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_holtin_finetuned_full_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/holtin/distilbert-base-uncased-holtin-finetuned-full-squad --- layout: model title: Embeddings Sciwiki 150 dims author: John Snow Labs name: embeddings_sciwiki_150d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-27 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_sciwiki_150d_es_2.5.0_2.4_1590609340084.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_sciwiki_150d_es_2.5.0_2.4_1590609340084.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_sciwiki_150d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_sciwiki_150d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.sciwiki.150d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_sciwiki_150d``. {:.model-param} ## Model Information {:.table-model} |---------------|-------------------------| | Name: | embeddings_sciwiki_150d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | es | | Dimension: | 150.0 | {:.h2_title} ## Data Source Trained on Clinical Wikipedia Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_128 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_128_zh_4.2.4_3.0_1670021708476.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_128_zh_4.2.4_3.0_1670021708476.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_128| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|15.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-128 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Sentence Embeddings - sbert mini (tuned) author: John Snow Labs name: sbert_jsl_mini_umls_uncased date: 2021-05-14 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_mini_umls_uncased_en_3.0.3_2.4_1621017142607.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_mini_umls_uncased_en_3.0.3_2.4_1621017142607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings\ .pretrained("sbert_jsl_mini_umls_uncased","en","clinical/models")\ .setInputCols(["sentence"])\ .setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_mini_umls_uncased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_mini_umlsuncased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_mini_umls_uncased| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|false| ## Data Source Tuned on MedNLI and UMLS dataset ## Benchmarking ```bash MedNLI Score Acc 0.677 STS(cos) 0.681 ``` --- layout: model title: Legal Custodian Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_custodian_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, custodian, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_custodian_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `custodian-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `custodian-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_custodian_agreement_bert_en_1.0.0_3.0_1669310583748.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_custodian_agreement_bert_en_1.0.0_3.0_1669310583748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_custodian_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[custodian-agreement]| |[other]| |[other]| |[custodian-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_custodian_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support custodian-agreement 0.98 0.93 0.95 43 other 0.96 0.99 0.98 82 accuracy - - 0.97 125 macro-avg 0.97 0.96 0.96 125 weighted-avg 0.97 0.97 0.97 125 ``` --- layout: model title: Italian DistilBertForMaskedLM Cased model (from indigo-ai) author: John Snow Labs name: distilbert_embeddings_bertino date: 2022-12-12 tags: [it, open_source, distilbert_embeddings, distilbertformaskedlm] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BERTino` is a Italian model originally trained by `indigo-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_bertino_it_4.2.4_3.0_1670864710883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_bertino_it_4.2.4_3.0_1670864710883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_bertino","it") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(False) pipeline = Pipeline(stages=[documentAssembler, tokenizer, distilbert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_bertino","it") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(false) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, distilbert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_bertino| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|it| |Size:|253.3 MB| |Case sensitive:|false| ## References - https://huggingface.co/indigo-ai/BERTino - https://indigo.ai/en/ - https://www.corpusitaliano.it/ - https://corpora.dipintra.it/public/run.cgi/corp_info?corpname=itwac_full - https://universaldependencies.org/treebanks/it_partut/index.html - https://universaldependencies.org/treebanks/it_isdt/index.html - https://figshare.com/articles/Learning_multilingual_named_entity_recognition_from_Wikipedia/5462500 --- layout: model title: Detect PHI for Deidentification (Augmented) author: John Snow Labs name: ner_deid_augmented date: 2021-01-20 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 2.7.1 spark_version: 2.4 tags: [en, deidentify, ner, clinical, licensed] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deidentification NER (Augmented) is a Named Entity Recognition model that annotates text to find protected health information that may need to be deidentified. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/3de6f25c23cd487d829ac3ce444ef19cfbe02631/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentificiation.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_augmented_en_2.7.1_2.4_1611145829422.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_augmented_en_2.7.1_2.4_1611145829422.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use This model is trained with the ‘embeddings_clinical’ word embeddings, so be sure to use the same embeddings within the pipeline in addition to document assembler, sentence detector, tokenizer and ner converter .

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = NerDLModel.pretrained("ner_deid_augmented","en","clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(['document', 'token', 'ner']) \ .setOutputCol('ner_chunk') nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same. ']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("ner_deid_augmented","en","clinical/models") .setInputCols(Array("sentence","token","word_embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token", "ner")) .setOutputCol("ner_chunk") val nlpPipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_converter)) val result = pipeline.fit(Seq.empty[String]).transform(data) val results = LightPipeline(model).fullAnnotate("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.augmented").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same. """) ```

## Results ```bash +---------------+---------+ |chunk |ner_label| +---------------+---------+ |Smith |NAME | |VA Hospital |LOCATION | |John Green |NAME | |2347165768 |ID | |Day Hospital |LOCATION | |02/04/2003 |DATE | |Smith |NAME | |Day Hospital |LOCATION | |Smith |NAME | |Smith |NAME | |7 Ardmore Tower|LOCATION | |Hart |NAME | |Smith |NAME | |02/07/2003 |DATE | +---------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_augmented| |Type:|ner| |Compatibility:|Spark NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on plain n2c2 2014: De-identification and Heart Disease Risk Factors Challenge datasets with embeddings_clinical https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/ ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | I-NAME | 1096 | 47 | 80 | 0.95888 | 0.931973 | 0.945235 | | 1 | I-CONTACT | 93 | 0 | 4 | 1 | 0.958763 | 0.978947 | | 2 | I-AGE | 3 | 1 | 6 | 0.75 | 0.333333 | 0.461538 | | 3 | B-DATE | 2078 | 42 | 52 | 0.980189 | 0.975587 | 0.977882 | | 4 | I-DATE | 474 | 39 | 25 | 0.923977 | 0.9499 | 0.936759 | | 5 | I-LOCATION | 755 | 68 | 76 | 0.917375 | 0.908544 | 0.912938 | | 6 | I-PROFESSION | 78 | 8 | 9 | 0.906977 | 0.896552 | 0.901734 | | 7 | B-NAME | 1182 | 101 | 36 | 0.921278 | 0.970443 | 0.945222 | | 8 | B-AGE | 259 | 10 | 11 | 0.962825 | 0.959259 | 0.961039 | | 9 | B-ID | 146 | 8 | 11 | 0.948052 | 0.929936 | 0.938907 | | 10 | B-PROFESSION | 76 | 9 | 21 | 0.894118 | 0.783505 | 0.835165 | | 11 | B-LOCATION | 556 | 87 | 71 | 0.864697 | 0.886762 | 0.875591 | | 12 | I-ID | 64 | 8 | 3 | 0.888889 | 0.955224 | 0.920863 | | 13 | B-CONTACT | 40 | 7 | 5 | 0.851064 | 0.888889 | 0.869565 | | 14 | Macro-average | 6900 | 435 | 410 | 0.912023 | 0.880619 | 0.896046 | | 15 | Micro-average | 6900 | 435 | 410 | 0.940695 | 0.943912 | 0.942301 | ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from nbroad) author: John Snow Labs name: roberta_qa_base_super_1 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rob-base-superqa1` is a English model originally trained by `nbroad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_super_1_en_4.3.0_3.0_1674212339330.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_super_1_en_4.3.0_3.0_1674212339330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_super_1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_super_1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_super_1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nbroad/rob-base-superqa1 - https://paperswithcode.com/sota?task=Question+Answering&dataset=adversarial_qa --- layout: model title: Fast Neural Machine Translation Model from Pedi to English author: John Snow Labs name: opus_mt_nso_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, nso, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `nso` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_nso_en_xx_2.7.0_2.4_1609170845142.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_nso_en_xx_2.7.0_2.4_1609170845142.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_nso_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Finance Capital Call Notices Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: finclf_capital_call_notices date: 2023-02-16 tags: [en, licensed, finance, capital_calls, classification, tensorflow] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `finclf_capital_call_notices` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `capital_call_notices` or not (Binary Classification). ## Predicted Entities `capital_call_notices`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_capital_call_notices_en_1.0.0_3.0_1676590287518.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_capital_call_notices_en_1.0.0_3.0_1676590287518.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = finance.ClassifierDLModel.pretrained("finclf_capital_call_notices", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[capital_call_notices]| |[other]| |[other]| |[capital_call_notices]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_capital_call_notices| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Financial documents and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support capital_call_notices 1.00 1.00 1.00 12 other 1.00 1.00 1.00 23 accuracy - - 1.00 35 macro-avg 1.00 1.00 1.00 35 weighted-avg 1.00 1.00 1.00 35 ``` --- layout: model title: Sentiment Analysis of IMDB Reviews Pipeline (analyze_sentimentdl_glove_imdb) author: John Snow Labs name: analyze_sentimentdl_glove_imdb date: 2021-01-15 task: [Embeddings, Sentiment Analysis, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [sentiment, en, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline to classify IMDB reviews in `neg` and `pos` classes using `glove_100d` embeddings. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_glove_imdb_en_2.7.1_2.4_1610722058784.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/analyze_sentimentdl_glove_imdb_en_2.7.1_2.4_1610722058784.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("analyze_sentimentdl_glove_imdb", lang = "en") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("analyze_sentimentdl_glove_imdb", lang = "en") ``` {:.nlu-block} ```python import nlu nlu.load("en.sentiment.glove").predict("""Put your text here.""") ```

## Results ```bash | | document | sentiment | |---:|---------------------------------------------------------------------------------------------------------:|--------------:| | | Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the | | | 0 | film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music | positive | | | was rad! Horror and sword fight freaks,buy this movie now! | | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|analyze_sentimentdl_glove_imdb| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Language:|en| ## Included Models `glove_100d`, `sentimentdl_glove_imdb` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from anurag0077) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_squad3 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad3` is a English model originally trained by `anurag0077`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad3_en_4.3.0_3.0_1672773665178.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad3_en_4.3.0_3.0_1672773665178.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_squad3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anurag0077/distilbert-base-uncased-finetuned-squad3 --- layout: model title: Translate Hiligaynon to English Pipeline author: John Snow Labs name: translate_hil_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, hil, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `hil` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_hil_en_xx_2.7.0_2.4_1609686032449.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_hil_en_xx_2.7.0_2.4_1609686032449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_hil_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_hil_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.hil.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_hil_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForTokenClassification Cased model (from ml6team) author: John Snow Labs name: distilbert_token_classifier_keyphrase_extraction_openkp date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyphrase-extraction-distilbert-openkp` is a English model originally trained by `ml6team`. ## Predicted Entities `KEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_openkp_en_4.3.0_3.0_1677880905122.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_openkp_en_4.3.0_3.0_1677880905122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_openkp","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_openkp","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_keyphrase_extraction_openkp| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ml6team/keyphrase-extraction-distilbert-openkp - https://github.com/microsoft/OpenKP - https://arxiv.org/abs/1911.02671 - https://paperswithcode.com/sota?task=Keyphrase+Extraction&dataset=openkp --- layout: model title: Word2Vec Embeddings in Bihari languages (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, bh, open_source] task: Embeddings language: bh edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_bh_3.4.1_3.0_1647286940542.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_bh_3.4.1_3.0_1647286940542.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ti_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ti_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ti.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ti_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Danish asr_alvenir_wav2vec2_base_nst_cv9 TFWav2Vec2ForCTC from chcaa author: John Snow Labs name: pipeline_asr_alvenir_wav2vec2_base_nst_cv9 date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_alvenir_wav2vec2_base_nst_cv9` is a Danish model originally trained by chcaa. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_alvenir_wav2vec2_base_nst_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_alvenir_wav2vec2_base_nst_cv9_da_4.2.0_3.0_1664104731248.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_alvenir_wav2vec2_base_nst_cv9_da_4.2.0_3.0_1664104731248.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_alvenir_wav2vec2_base_nst_cv9', lang = 'da') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_alvenir_wav2vec2_base_nst_cv9", lang = "da") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_alvenir_wav2vec2_base_nst_cv9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|da| |Size:|226.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Thai to English Pipeline author: John Snow Labs name: translate_th_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, th, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `th` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_th_en_xx_2.7.0_2.4_1609689519812.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_th_en_xx_2.7.0_2.4_1609689519812.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_th_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_th_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.th.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_th_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Diseases in Medical Text author: John Snow Labs name: bert_token_classifier_ner_bc5cdr_disease_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_bc5cdr_disease](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_bc5cdr_disease_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc5cdr_disease_pipeline_en_4.3.0_3.2_1679302082722.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc5cdr_disease_pipeline_en_4.3.0_3.2_1679302082722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_bc5cdr_disease_pipeline", "en", "clinical/models") text = '''Indomethacin resulted in histopathologic findings typical of interstitial cystitis, such as leaky bladder epithelium and mucosal mastocytosis. The true incidence of nonsteroidal anti-inflammatory drug-induced cystitis in humans must be clarified by prospective clinical trials. An open-label phase II study of low-dose thalidomide in androgen-independent prostate cancer.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_bc5cdr_disease_pipeline", "en", "clinical/models") val text = "Indomethacin resulted in histopathologic findings typical of interstitial cystitis, such as leaky bladder epithelium and mucosal mastocytosis. The true incidence of nonsteroidal anti-inflammatory drug-induced cystitis in humans must be clarified by prospective clinical trials. An open-label phase II study of low-dose thalidomide in androgen-independent prostate cancer." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:------------|-------------:| | 0 | interstitial cystitis | 61 | 81 | DISEASE | 0.999746 | | 1 | mastocytosis | 129 | 140 | DISEASE | 0.999132 | | 2 | cystitis | 209 | 216 | DISEASE | 0.999912 | | 3 | prostate cancer | 355 | 369 | DISEASE | 0.999781 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bc5cdr_disease_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English BertForQuestionAnswering model (from LoudlySoft) author: John Snow Labs name: bert_qa_scibert_scivocab_uncased_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `scibert_scivocab_uncased_squad` is a English model orginally trained by `LoudlySoft`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_scivocab_uncased_squad_en_4.0.0_3.0_1654189441461.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_scivocab_uncased_squad_en_4.0.0_3.0_1654189441461.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_scibert_scivocab_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_scibert_scivocab_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.scibert.uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_scibert_scivocab_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/LoudlySoft/scibert_scivocab_uncased_squad --- layout: model title: Financial Finetuned FLAN-T5 Text Generation ( Financial Alpaca ) author: John Snow Labs name: fingen_flant5_finetuned_alpaca date: 2023-05-25 tags: [en, finance, generation, licensed, flant5, alpaca, tensorflow] task: Text Generation language: en edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: FinanceTextGenerator article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `fingen_flant5_finetuned_alpaca` model is the Text Generation model that has been fine-tuned on FLAN-T5 using Financial Alpaca dataset. FLAN-T5 is a state-of-the-art language model developed by Facebook AI that utilizes the T5 architecture for text-generation tasks. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/fingen_flant5_finetuned_alpaca_en_1.0.0_3.0_1685016665729.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/fingen_flant5_finetuned_alpaca_en_1.0.0_3.0_1685016665729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") flant5 = finance.TextGenerator.pretrained("fingen_flant5_finetuned_alpaca", "en", "finance/models")\ .setInputCols(["document"])\ .setOutputCol("generated")\ .setMaxNewTokens(256)\ .setStopAtEos(True)\ .setDoSample(True)\ .setTopK(3) pipeline = nlp.Pipeline(stages=[document_assembler, flant5]) data = spark.createDataFrame([ [1, "What is the US Fair Tax?"]]).toDF('id', 'text') results = pipeline.fit(data).transform(data) results.select("generated.result").show(truncate=False) ```

## Results ```bash +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |result | +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[Fair tax in the US is essentially an income tax. Fair taxes are tax on your income, and are not taxeable in any country. Fair taxes are taxed as income. If you have a net gain or if the loss of income from taxable activities is less then the fair value (the loss) of your gross income (the loss) then you have to file an Income Report. This will give the US government an overview and give you an understanding. If your net income is less that your fair share of your gross income (which you are entitled) you have the right to claim a refund.]| +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|fingen_flant5_finetuned_alpaca| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.6 GB| ## References The dataset is available [here](https://huggingface.co/datasets/gbharti/finance-alpaca/viewer/gbharti--finance-alpaca) --- layout: model title: English Deberta Embeddings model (from ZZ99) author: John Snow Labs name: deberta_embeddings_tapt_nbme_v3_base date: 2023-03-13 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, en, tensorflow] task: Embeddings language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tapt_nbme_deberta_v3_base` is a English model originally trained by `ZZ99`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_tapt_nbme_v3_base_en_4.3.1_3.0_1678712713960.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_tapt_nbme_v3_base_en_4.3.1_3.0_1678712713960.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_tapt_nbme_v3_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_tapt_nbme_v3_base","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_tapt_nbme_v3_base| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|689.5 MB| |Case sensitive:|false| ## References https://huggingface.co/ZZ99/tapt_nbme_deberta_v3_base --- layout: model title: Detect Living Species (bert_base_cased) author: John Snow Labs name: ner_living_species_bert date: 2022-06-23 tags: [ro, ner, clinical, licensed, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Romanian which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `bert_base_cased` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_ro_3.5.3_3.0_1655974560466.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_ro_3.5.3_3.0_1655974560466.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_base_cased", "ro")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "ro", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""O femeie în vârstă de 26 de ani, însărcinată în 11 săptămâni, a consultat serviciul de urgențe dermatologice pentru că prezenta, de 4 zile, leziuni punctiforme dureroase de debut brusc pe vârful degetelor. Pacientul raportează că leziunile au început pe degete și ulterior s-au extins la degetele de la picioare. Markerii de imunitate, ANA și crioagglutininele, au fost negativi, iar serologia VHB a indicat doar vaccinarea. Pe baza acestor rezultate, diagnosticul de vasculită a fost exclus și, având în vedere diagnosticul suspectat de erupție cutanată cu mănuși și șosete, s-a efectuat serologia pentru virusul Ebstein Barr. Exantemă la mănuși și șosete datorat parvovirozei B19. Având în vedere suspiciunea unei afecțiuni infecțioase cu aceste caracteristici, a fost solicitată serologia pentru EBV, enterovirus și parvovirus B19, cu IgM pozitiv pentru acesta din urmă în două ocazii. De asemenea, nu au existat semne de anemie fetală sau complicații ale acesteia."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_base_cased", "ro") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "ro", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""O femeie în vârstă de 26 de ani, însărcinată în 11 săptămâni, a consultat serviciul de urgențe dermatologice pentru că prezenta, de 4 zile, leziuni punctiforme dureroase de debut brusc pe vârful degetelor. Pacientul raportează că leziunile au început pe degete și ulterior s-au extins la degetele de la picioare. Markerii de imunitate, ANA și crioagglutininele, au fost negativi, iar serologia VHB a indicat doar vaccinarea. Pe baza acestor rezultate, diagnosticul de vasculită a fost exclus și, având în vedere diagnosticul suspectat de erupție cutanată cu mănuși și șosete, s-a efectuat serologia pentru virusul Ebstein Barr. Exantemă la mănuși și șosete datorat parvovirozei B19. Având în vedere suspiciunea unei afecțiuni infecțioase cu aceste caracteristici, a fost solicitată serologia pentru EBV, enterovirus și parvovirus B19, cu IgM pozitiv pentru acesta din urmă în două ocazii. De asemenea, nu au existat semne de anemie fetală sau complicații ale acesteia.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ro.med_ner.living_species.bert").predict("""O femeie în vârstă de 26 de ani, însărcinată în 11 săptămâni, a consultat serviciul de urgențe dermatologice pentru că prezenta, de 4 zile, leziuni punctiforme dureroase de debut brusc pe vârful degetelor. Pacientul raportează că leziunile au început pe degete și ulterior s-au extins la degetele de la picioare. Markerii de imunitate, ANA și crioagglutininele, au fost negativi, iar serologia VHB a indicat doar vaccinarea. Pe baza acestor rezultate, diagnosticul de vasculită a fost exclus și, având în vedere diagnosticul suspectat de erupție cutanată cu mănuși și șosete, s-a efectuat serologia pentru virusul Ebstein Barr. Exantemă la mănuși și șosete datorat parvovirozei B19. Având în vedere suspiciunea unei afecțiuni infecțioase cu aceste caracteristici, a fost solicitată serologia pentru EBV, enterovirus și parvovirus B19, cu IgM pozitiv pentru acesta din urmă în două ocazii. De asemenea, nu au existat semne de anemie fetală sau complicații ale acesteia.""") ```

## Results ```bash +--------------------+-------+ |ner_chunk |label | +--------------------+-------+ |femeie |HUMAN | |Pacientul |HUMAN | |VHB |SPECIES| |virusul Ebstein Barr|SPECIES| |parvovirozei B19 |SPECIES| |EBV |SPECIES| |enterovirus |SPECIES| |parvovirus B19 |SPECIES| |fetală |HUMAN | +--------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.4 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.85 0.94 0.89 2184 B-SPECIES 0.75 0.85 0.80 2617 I-HUMAN 0.89 0.11 0.20 72 I-SPECIES 0.74 0.80 0.77 1027 micro-avg 0.79 0.86 0.82 5900 macro-avg 0.81 0.67 0.66 5900 weighted-avg 0.79 0.86 0.82 5900 ``` --- layout: model title: English DistilBertForTokenClassification Cased model (from ml6team) author: John Snow Labs name: distilbert_token_classifier_keyphrase_extraction_openkp date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyphrase-extraction-distilbert-openkp` is a English model originally trained by `ml6team`. ## Predicted Entities `KEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_openkp_en_4.3.1_3.0_1678782889694.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_keyphrase_extraction_openkp_en_4.3.1_3.0_1678782889694.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_openkp","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_openkp","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_keyphrase_extraction_openkp| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ml6team/keyphrase-extraction-distilbert-openkp - https://github.com/microsoft/OpenKP - https://arxiv.org/abs/1911.02671 - https://paperswithcode.com/sota?task=Keyphrase+Extraction&dataset=openkp --- layout: model title: English image_classifier_vit_vision_transformer_fmri_classification_ft ViTForImageClassification from shivkumarganesh author: John Snow Labs name: image_classifier_vit_vision_transformer_fmri_classification_ft date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vision_transformer_fmri_classification_ft` is a English model originally trained by shivkumarganesh. ## Predicted Entities `test`, `train`, `val` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformer_fmri_classification_ft_en_4.1.0_3.0_1660166000402.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformer_fmri_classification_ft_en_4.1.0_3.0_1660166000402.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vision_transformer_fmri_classification_ft", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vision_transformer_fmri_classification_ft", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vision_transformer_fmri_classification_ft| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Special terms and conditions of trust Clause Binary Classifier author: John Snow Labs name: legclf_special_terms_and_conditions_of_trust_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `special-terms-and-conditions-of-trust` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `special-terms-and-conditions-of-trust` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_special_terms_and_conditions_of_trust_clause_en_1.0.0_3.2_1660123013530.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_special_terms_and_conditions_of_trust_clause_en_1.0.0_3.2_1660123013530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_special_terms_and_conditions_of_trust_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[special-terms-and-conditions-of-trust]| |[other]| |[other]| |[special-terms-and-conditions-of-trust]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_special_terms_and_conditions_of_trust_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 132 special-terms-and-conditions-of-trust 1.00 1.00 1.00 56 accuracy - - 1.00 188 macro-avg 1.00 1.00 1.00 188 weighted-avg 1.00 1.00 1.00 188 ``` --- layout: model title: Movies Sentiment Analysis author: John Snow Labs name: movies_sentiment_analysis date: 2022-07-06 tags: [en, open_source] task: Sentiment Analysis language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The movies_sentiment_analysis is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and predicts sentiment . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/movies_sentiment_analysis_en_4.0.0_3.0_1657135804995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/movies_sentiment_analysis_en_4.0.0_3.0_1657135804995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("movies_sentiment_analysis", "en") result = pipeline.annotate("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|movies_sentiment_analysis| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|210.0 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - SymmetricDeleteModel - SentimentDetectorModel --- layout: model title: Word2Vec Embeddings in Galician (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, gl, open_source] task: Embeddings language: gl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_gl_3.4.1_3.0_1647374243984.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_gl_3.4.1_3.0_1647374243984.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_afriberta_small| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|311.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/castorini/afriberta_small - https://github.com/keleog/afriberta --- layout: model title: Smaller BERT Sentence Embeddings (L-6_H-256_A-4) author: John Snow Labs name: sent_small_bert_L6_256 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L6_256_en_2.6.0_2.4_1598350409969.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L6_256_en_2.6.0_2.4_1598350409969.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L6_256", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L6_256", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L6_256').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L6_256_embeddings sentence [0.7711525559425354, 0.5496315956115723, 1.261... I hate cancer [0.28574034571647644, -0.03116176463663578, 1.... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L6_256| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1 --- layout: model title: Legal Injunctive relief Clause Binary Classifier author: John Snow Labs name: legclf_injunctive_relief_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `injunctive-relief` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `injunctive-relief` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_injunctive_relief_clause_en_1.0.0_3.2_1660122542368.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_injunctive_relief_clause_en_1.0.0_3.2_1660122542368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_injunctive_relief_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[injunctive-relief]| |[other]| |[other]| |[injunctive-relief]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_injunctive_relief_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support injunctive-relief 0.91 1.00 0.95 30 other 1.00 0.97 0.99 103 accuracy - - 0.98 133 macro-avg 0.95 0.99 0.97 133 weighted-avg 0.98 0.98 0.98 133 ``` --- layout: model title: Translate English to Hiri Motu Pipeline author: John Snow Labs name: translate_en_ho date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ho, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ho` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ho_xx_2.7.0_2.4_1609691430837.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ho_xx_2.7.0_2.4_1609691430837.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ho", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ho", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ho').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ho| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Full disclosure Clause Binary Classifier author: John Snow Labs name: legclf_full_disclosure_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `full-disclosure` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `full-disclosure` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_full_disclosure_clause_en_1.0.0_3.2_1660122467527.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_full_disclosure_clause_en_1.0.0_3.2_1660122467527.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_full_disclosure_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[full-disclosure]| |[other]| |[other]| |[full-disclosure]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_full_disclosure_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support full-disclosure 1.00 0.94 0.97 31 other 0.98 1.00 0.99 104 accuracy - - 0.99 135 macro-avg 0.99 0.97 0.98 135 weighted-avg 0.99 0.99 0.99 135 ``` --- layout: model title: Pipeline to Summarize Clinical Question Notes author: John Snow Labs name: summarizer_clinical_questions_pipeline date: 2023-05-29 tags: [licensed, en, clinical, summarization, question] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_questions](https://nlp.johnsnowlabs.com/2023/04/03/summarizer_clinical_questions_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_pipeline_en_4.4.2_3.0_1685401048463.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_pipeline_en_4.4.2_3.0_1685401048463.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_questions_pipeline", "en", "clinical/models") text = """ Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_questions_pipeline", "en", "clinical/models") val text = """ Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash What are the treatments for hyperthyroidism? ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_questions_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|936.7 MB| ## Included Models - DocumentAssembler - MedicalSummarizer --- layout: model title: Chinese BertForQuestionAnswering model (from uer) author: John Snow Labs name: bert_qa_roberta_base_chinese_extractive_qa date: 2022-06-02 tags: [zh, open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-chinese-extractive-qa` is a Chinese model orginally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_qa_zh_4.0.0_3.0_1654189258198.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_qa_zh_4.0.0_3.0_1654189258198.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_roberta_base_chinese_extractive_qa","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_roberta_base_chinese_extractive_qa","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.base.by_uer").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_roberta_base_chinese_extractive_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|381.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/uer/roberta-base-chinese-extractive-qa - https://spaces.ac.cn/archives/4338 - https://www.kesci.com/home/competition/5d142d8cbb14e6002c04e14a/content/0 - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/product/tione/ - https://github.com/ymcui/cmrc2018 --- layout: model title: English RobertaForQuestionAnswering (from nlpconnect) author: John Snow Labs name: roberta_qa_roberta_base_squad2_nq date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-nq` is a English model originally trained by `nlpconnect`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_nq_en_4.0.0_3.0_1655735618263.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_nq_en_4.0.0_3.0_1655735618263.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_squad2_nq","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_squad2_nq","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_nlpconnect").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_squad2_nq| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpconnect/roberta-base-squad2-nq --- layout: model title: German T5ForConditionalGeneration Base Cased model (from Einmalumdiewelt) author: John Snow Labs name: t5_base_gnad_maxsamples date: 2023-01-30 tags: [de, open_source, t5, tensorflow] task: Text Generation language: de edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `T5-Base_GNAD_MaxSamples` is a German model originally trained by `Einmalumdiewelt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_gnad_maxsamples_de_4.3.0_3.0_1675099257674.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_gnad_maxsamples_de_4.3.0_3.0_1675099257674.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_gnad_maxsamples","de") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_gnad_maxsamples","de") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_gnad_maxsamples| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|de| |Size:|922.8 MB| ## References - https://huggingface.co/Einmalumdiewelt/T5-Base_GNAD_MaxSamples --- layout: model title: Context Spell Checker for the English Language author: John Snow Labs name: spellcheck_dl date: 2021-03-28 tags: [en, open_source] supported: true task: Spell Check language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: ContextSpellCheckerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Spell Checker is a sequence-to-sequence model that detects and corrects spelling errors in your input text. It’s based on Levenshtein Automaton for generating candidate corrections and a Neural Language Model for ranking corrections. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_en_3.0.0_3.0_1616900699393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_en_3.0.0_3.0_1616900699393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use The model works at the token level, so you must put it after tokenization. The model can change the length of the tokens when correcting words, so keep this in mind when using it before other annotators that may work with absolute references to the original document like NerConverter.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = RecursiveTokenizer()\ .setInputCols(["document"])\ .setOutputCol("token")\ .setPrefixes(["\"", "“", "(", "[", "\n", "."]) \ .setSuffixes(["\"", "”", ".", ",", "?", ")", "]", "!", ";", ":", "'s", "’s"]) spellModel = ContextSpellCheckerModel\ .pretrained()\ .setInputCols("token")\ .setOutputCol("checked")\ ``` ```scala val assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new RecursiveTokenizer() .setInputCols(Array("document")) .setOutputCol("token") .setPrefixes(Array("\"", "“", "(", "[", "\n", ".")) .setSuffixes(Array("\"", "”", ".", ",", "?", ")", "]", "!", ";", ":", "'s", "’s")) val spellChecker = ContextSpellCheckerModel. pretrained(). setInputCols("token"). setOutputCol("checked") ``` {:.nlu-block} ```python import nlu nlu.load("spell").predict("""]) \ .setSuffixes([""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_dl| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[corrected]| |Language:|en| ## Data Source American National Corpus. --- layout: model title: English image_classifier_vit_modeversion28_7 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_modeversion28_7 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_modeversion28_7` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion28_7_en_4.1.0_3.0_1660168471234.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion28_7_en_4.1.0_3.0_1660168471234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_modeversion28_7", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_modeversion28_7", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_modeversion28_7| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: BioBERT Embeddings (Discharge) author: John Snow Labs name: biobert_discharge_base_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for discharge summaries. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_discharge_base_cased_en_2.6.2_2.4_1600531401858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_discharge_base_cased_en_2.6.2_2.4_1600531401858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("biobert_discharge_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("biobert_discharge_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.discharge_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_biobert_discharge_base_cased_embeddings I [0.0036486536264419556, 0.3796533942222595, -0... hate [0.1914958357810974, 0.6709488034248352, -0.49... cancer [0.04618441313505173, -0.04562612622976303, -0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_discharge_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.2| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: Dispute Clause Binary Classifier author: John Snow Labs name: legclf_dispute_clauses_cuad date: 2023-01-18 tags: [en, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `dispute_clause` clause type. To use this model, make sure you provide enough context as an input. Senteces have been used as positive examples, so better results will be achieved if SetenceDetector is added to the pipeline. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other 300+ Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `dispute_clause`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_dispute_clauses_cuad_en_1.0.0_3.0_1674056674986.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_dispute_clauses_cuad_en_1.0.0_3.0_1674056674986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['dispute_clause'] ['other'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_dispute_clauses_cuad| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[label]| |Language:|en| |Size:|22.9 MB| ## References Manual annotations of CUAD dataset ## Benchmarking ```bash label precision recall f1-score support dispute_clause 1.00 1.00 1.00 61 other 1.00 1.00 1.00 96 accuracy - - 1.00 157 macro-avg 1.00 1.00 1.00 157 weighted-avg 1.00 1.00 1.00 157 ``` --- layout: model title: Pipeline to Detect Problems, Tests and Treatments author: John Snow Labs name: ner_healthcare_pipeline date: 2023-03-14 tags: [ner, licensed, clinical, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_healthcare](https://nlp.johnsnowlabs.com/2021/04/21/ner_healthcare_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_pipeline_en_4.3.0_3.2_1678824932575.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_pipeline_en_4.3.0_3.2_1678824932575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_healthcare_pipeline", "en", "clinical/models") text = '''A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_healthcare_pipeline", "en", "clinical/models") val text = "A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG ." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.healthcare_pipeline").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:------------|-------------:| | 0 | gestational diabetes mellitus | 39 | 67 | PROBLEM | 0.938233 | | 1 | type two diabetes mellitus | 128 | 153 | PROBLEM | 0.762925 | | 2 | HTG-induced pancreatitis | 186 | 209 | PROBLEM | 0.9742 | | 3 | an acute hepatitis | 263 | 280 | PROBLEM | 0.915067 | | 4 | obesity | 288 | 294 | PROBLEM | 0.9926 | | 5 | a body mass index | 301 | 317 | TEST | 0.721175 | | 6 | BMI | 321 | 323 | TEST | 0.4466 | | 7 | polyuria | 380 | 387 | PROBLEM | 0.9987 | | 8 | polydipsia | 391 | 400 | PROBLEM | 0.9993 | | 9 | poor appetite | 404 | 416 | PROBLEM | 0.96315 | | 10 | vomiting | 424 | 431 | PROBLEM | 0.9588 | | 11 | amoxicillin | 511 | 521 | TREATMENT | 0.6453 | | 12 | a respiratory tract infection | 527 | 555 | PROBLEM | 0.867 | | 13 | metformin | 570 | 578 | TREATMENT | 0.9989 | | 14 | glipizide | 582 | 590 | TREATMENT | 0.9997 | | 15 | dapagliflozin | 598 | 610 | TREATMENT | 0.9996 | | 16 | T2DM | 616 | 619 | TREATMENT | 0.9662 | | 17 | atorvastatin | 625 | 636 | TREATMENT | 0.9993 | | 18 | gemfibrozil | 642 | 652 | TREATMENT | 0.9997 | | 19 | HTG | 658 | 660 | PROBLEM | 0.9927 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English ElectraForQuestionAnswering model (from mrm8488) Version-2 author: John Snow Labs name: electra_qa_base_finetuned_squadv2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-finetuned-squadv2` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_finetuned_squadv2_en_4.0.0_3.0_1655920687688.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_finetuned_squadv2_en_4.0.0_3.0_1655920687688.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_finetuned_squadv2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.electra.base_v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/electra-base-finetuned-squadv2 --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes author: John Snow Labs name: sbiobertresolve_umls_major_concepts date: 2021-05-02 tags: [en, clinical, licensed, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Map clinical entities to UMLS CUI codes. ## Predicted Entities This model returns CUI (concept unique identifier) codes for `Clinical Findings`, `Medical Devices`, `Anatomical Structures` and `Injuries & Poisoning` terms {:.btn-box} [Live Demo](http://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_major_concepts_en_3.0.2_3.0_1619973285528.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_major_concepts_en_3.0.2_3.0_1619973285528.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_umls_major_concepts``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Cerebrovascular_Disease, Communicable_Disease, Diabetes, Disease_Syndrome_Disorder, Heart_Disease, Hyperlipidemia, Hypertension, Injury_or_Poisoning, Kidney_Disease, Medical-Device, Obesity, Oncological, Overweight, Psychological_Condition, Symptom, VS_Finding, ImagingFindings, EKG_Findings``` set in ```.setWhiteList()```.

## Results ```bash | | ner_chunk | resolution | |---:|:------------------------------|:-------------| | 0 | 28-year-old | C1864118 | | 1 | female | C3887375 | | 2 | gestational diabetes mellitus | C2183115 | | 3 | eight years prior | C5195266 | | 4 | subsequent | C3844350 | | 5 | type two diabetes mellitus | C4014362 | | 6 | T2DM | C4014362 | | 7 | HTG-induced pancreatitis | C4554179 | | 8 | three years prior | C1866782 | | 9 | acute | C1332147 | | 10 | hepatitis | C1963279 | | 11 | obesity | C1963185 | | 12 | body mass index | C0578022 | | 13 | 33.5 kg/m2 | C2911054 | | 14 | one-week | C0420331 | | 15 | polyuria | C3278312 | | 16 | polydipsia | C3278316 | | 17 | poor appetite | C0541799 | | 18 | vomiting | C1963281 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_major_concepts| |Compatibility:|Healthcare NLP 3.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[umls_code]| |Language:|en| ## Data Source https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: English image_classifier_vit_lucky_model ViTForImageClassification from Loc author: John Snow Labs name: image_classifier_vit_lucky_model date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_lucky_model` is a English model originally trained by Loc. ## Predicted Entities `turnstile`, `damselfly`, `mixing bowl`, `sea snake`, `cockroach, roach`, `buckle`, `beer glass`, `bulbul`, `lumbermill, sawmill`, `whippet`, `Australian terrier`, `television, television system`, `hoopskirt, crinoline`, `horse cart, horse-cart`, `guillotine`, `malamute, malemute, Alaskan malamute`, `coyote, prairie wolf, brush wolf, Canis latrans`, `colobus, colobus monkey`, `hognose snake, puff adder, sand viper`, `sock`, `burrito`, `printer`, `bathing cap, swimming cap`, `chiton, coat-of-mail shell, sea cradle, polyplacophore`, `Rottweiler`, `cello, violoncello`, `pitcher, ewer`, `computer keyboard, keypad`, `bow`, `peacock`, `ballplayer, baseball player`, `refrigerator, icebox`, `solar dish, solar collector, solar furnace`, `passenger car, coach, carriage`, `African chameleon, Chamaeleo chamaeleon`, `oboe, hautboy, hautbois`, `toyshop`, `Leonberg`, `howler monkey, howler`, `bluetick`, `African elephant, Loxodonta africana`, `American lobster, Northern lobster, Maine lobster, Homarus americanus`, `combination lock`, `black-and-tan coonhound`, `bonnet, poke bonnet`, `harvester, reaper`, `Appenzeller`, `iron, smoothing iron`, `electric locomotive`, `lycaenid, lycaenid butterfly`, `sandbar, sand bar`, `Cardigan, Cardigan Welsh corgi`, `pencil sharpener`, `jean, blue jean, denim`, `backpack, back pack, knapsack, packsack, rucksack, haversack`, `monitor`, `ice cream, icecream`, `apiary, bee house`, `water jug`, `American coot, marsh hen, mud hen, water hen, Fulica americana`, `ground beetle, carabid beetle`, `jigsaw puzzle`, `ant, emmet, pismire`, `wreck`, `kuvasz`, `gyromitra`, `Ibizan hound, Ibizan Podenco`, `brown bear, bruin, Ursus arctos`, `bolo tie, bolo, bola tie, bola`, `Pembroke, Pembroke Welsh corgi`, `French bulldog`, `prison, prison house`, `ballpoint, ballpoint pen, ballpen, Biro`, `stage`, `airliner`, `dogsled, dog sled, dog sleigh`, `redshank, Tringa totanus`, `menu`, `Indian cobra, Naja naja`, `swab, swob, mop`, `window screen`, `brain coral`, `artichoke, globe artichoke`, `loupe, jeweler's loupe`, `loudspeaker, speaker, speaker unit, loudspeaker system, speaker system`, `panpipe, pandean pipe, syrinx`, `wok`, `croquet ball`, `plate`, `scoreboard`, `Samoyed, Samoyede`, `ocarina, sweet potato`, `beaver`, `borzoi, Russian wolfhound`, `horizontal bar, high bar`, `stretcher`, `seat belt, seatbelt`, `obelisk`, `forklift`, `feather boa, boa`, `frying pan, frypan, skillet`, `barbershop`, `hamper`, `face powder`, `Siamese cat, Siamese`, `ladle`, `dingo, warrigal, warragal, Canis dingo`, `mountain tent`, `head cabbage`, `echidna, spiny anteater, anteater`, `Polaroid camera, Polaroid Land camera`, `dumbbell`, `espresso`, `notebook, notebook computer`, `Norfolk terrier`, `binoculars, field glasses, opera glasses`, `carpenter's kit, tool kit`, `moving van`, `catamaran`, `tiger beetle`, `bikini, two-piece`, `Siberian husky`, `studio couch, day bed`, `bulletproof vest`, `lawn mower, mower`, `promontory, headland, head, foreland`, `soap dispenser`, `vulture`, `dam, dike, dyke`, `brambling, Fringilla montifringilla`, `toilet tissue, toilet paper, bathroom tissue`, `ringlet, ringlet butterfly`, `tiger cat`, `mobile home, manufactured home`, `Norwich terrier`, `little blue heron, Egretta caerulea`, `English setter`, `Tibetan mastiff`, `rocking chair, rocker`, `mask`, `maze, labyrinth`, `bookcase`, `viaduct`, `sweatshirt`, `plow, plough`, `basenji`, `typewriter keyboard`, `Windsor tie`, `coral fungus`, `desktop computer`, `Kerry blue terrier`, `Angora, Angora rabbit`, `can opener, tin opener`, `shield, buckler`, `triumphal arch`, `horned viper, cerastes, sand viper, horned asp, Cerastes cornutus`, `miniature schnauzer`, `tape player`, `jaguar, panther, Panthera onca, Felis onca`, `hook, claw`, `file, file cabinet, filing cabinet`, `chime, bell, gong`, `shower curtain`, `window shade`, `acoustic guitar`, `gas pump, gasoline pump, petrol pump, island dispenser`, `cicada, cicala`, `Petri dish`, `paintbrush`, `banana`, `chickadee`, `mountain bike, all-terrain bike, off-roader`, `lighter, light, igniter, ignitor`, `oil filter`, `cab, hack, taxi, taxicab`, `Christmas stocking`, `rugby ball`, `black widow, Latrodectus mactans`, `bustard`, `fiddler crab`, `web site, website, internet site, site`, `chocolate sauce, chocolate syrup`, `chainlink fence`, `fireboat`, `cocktail shaker`, `airship, dirigible`, `projectile, missile`, `bagel, beigel`, `screwdriver`, `oystercatcher, oyster catcher`, `pot, flowerpot`, `water bottle`, `Loafer`, `drumstick`, `soccer ball`, `cairn, cairn terrier`, `padlock`, `tow truck, tow car, wrecker`, `bloodhound, sleuthhound`, `punching bag, punch bag, punching ball, punchball`, `great grey owl, great gray owl, Strix nebulosa`, `scale, weighing machine`, `trench coat`, `briard`, `cheetah, chetah, Acinonyx jubatus`, `entertainment center`, `Boston bull, Boston terrier`, `Arabian camel, dromedary, Camelus dromedarius`, `steam locomotive`, `coil, spiral, volute, whorl, helix`, `plane, carpenter's plane, woodworking plane`, `gondola`, `spider web, spider's web`, `bathtub, bathing tub, bath, tub`, `pelican`, `miniature poodle`, `cowboy boot`, `perfume, essence`, `lakeside, lakeshore`, `timber wolf, grey wolf, gray wolf, Canis lupus`, `moped`, `sunscreen, sunblock, sun blocker`, `Brabancon griffon`, `puffer, pufferfish, blowfish, globefish`, `lifeboat`, `pool table, billiard table, snooker table`, `Bouvier des Flandres, Bouviers des Flandres`, `Pomeranian`, `theater curtain, theatre curtain`, `marimba, xylophone`, `baboon`, `vacuum, vacuum cleaner`, `pill bottle`, `pick, plectrum, plectron`, `hen`, `American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier`, `digital watch`, `pier`, `oxygen mask`, `Tibetan terrier, chrysanthemum dog`, `ostrich, Struthio camelus`, `water ouzel, dipper`, `drilling platform, offshore rig`, `magnetic compass`, `throne`, `butternut squash`, `minibus`, `EntleBucher`, `carousel, carrousel, merry-go-round, roundabout, whirligig`, `hot pot, hotpot`, `rain barrel`, `wood rabbit, cottontail, cottontail rabbit`, `miniature pinscher`, `partridge`, `three-toed sloth, ai, Bradypus tridactylus`, `English springer, English springer spaniel`, `corkscrew, bottle screw`, `fur coat`, `robin, American robin, Turdus migratorius`, `dowitcher`, `ruddy turnstone, Arenaria interpres`, `water snake`, `stove`, `Great Pyrenees`, `soft-coated wheaten terrier`, `carbonara`, `snail`, `breastplate, aegis, egis`, `wolf spider, hunting spider`, `hatchet`, `CD player`, `axolotl, mud puppy, Ambystoma mexicanum`, `pomegranate`, `poncho`, `leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea`, `lorikeet`, `spatula`, `jay`, `platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus`, `stethoscope`, `flagpole, flagstaff`, `coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch`, `agama`, `red wolf, maned wolf, Canis rufus, Canis niger`, `beaker`, `eft`, `pretzel`, `brassiere, bra, bandeau`, `frilled lizard, Chlamydosaurus kingi`, `joystick`, `goldfish, Carassius auratus`, `fig`, `maypole`, `caldron, cauldron`, `admiral`, `impala, Aepyceros melampus`, `spotted salamander, Ambystoma maculatum`, `syringe`, `hog, pig, grunter, squealer, Sus scrofa`, `handkerchief, hankie, hanky, hankey`, `tarantula`, `cheeseburger`, `pinwheel`, `sax, saxophone`, `dung beetle`, `broccoli`, `cassette player`, `milk can`, `traffic light, traffic signal, stoplight`, `shovel`, `sarong`, `tabby, tabby cat`, `parallel bars, bars`, `ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle`, `quill, quill pen`, `giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca`, `steel drum`, `quail`, `Blenheim spaniel`, `wig`, `hamster`, `ice lolly, lolly, lollipop, popsicle`, `seashore, coast, seacoast, sea-coast`, `chest`, `worm fence, snake fence, snake-rail fence, Virginia fence`, `missile`, `beer bottle`, `yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum`, `breakwater, groin, groyne, mole, bulwark, seawall, jetty`, `white wolf, Arctic wolf, Canis lupus tundrarum`, `guacamole`, `porcupine, hedgehog`, `trolleybus, trolley coach, trackless trolley`, `greenhouse, nursery, glasshouse`, `trimaran`, `Italian greyhound`, `potter's wheel`, `jacamar`, `wallet, billfold, notecase, pocketbook`, `Lakeland terrier`, `green lizard, Lacerta viridis`, `indigo bunting, indigo finch, indigo bird, Passerina cyanea`, `green mamba`, `walking stick, walkingstick, stick insect`, `crossword puzzle, crossword`, `eggnog`, `barrow, garden cart, lawn cart, wheelbarrow`, `remote control, remote`, `bicycle-built-for-two, tandem bicycle, tandem`, `wool, woolen, woollen`, `black grouse`, `abaya`, `marmoset`, `golf ball`, `jeep, landrover`, `Mexican hairless`, `dishwasher, dish washer, dishwashing machine`, `jersey, T-shirt, tee shirt`, `planetarium`, `goose`, `mailbox, letter box`, `capuchin, ringtail, Cebus capucinus`, `marmot`, `orangutan, orang, orangutang, Pongo pygmaeus`, `coffeepot`, `ambulance`, `shopping basket`, `pop bottle, soda bottle`, `red fox, Vulpes vulpes`, `crash helmet`, `street sign`, `affenpinscher, monkey pinscher, monkey dog`, `Arctic fox, white fox, Alopex lagopus`, `sidewinder, horned rattlesnake, Crotalus cerastes`, `ruffed grouse, partridge, Bonasa umbellus`, `muzzle`, `measuring cup`, `canoe`, `reflex camera`, `fox squirrel, eastern fox squirrel, Sciurus niger`, `French loaf`, `killer whale, killer, orca, grampus, sea wolf, Orcinus orca`, `dial telephone, dial phone`, `thimble`, `bubble`, `vizsla, Hungarian pointer`, `running shoe`, `mailbag, postbag`, `radio telescope, radio reflector`, `piggy bank, penny bank`, `Chihuahua`, `chambered nautilus, pearly nautilus, nautilus`, `Airedale, Airedale terrier`, `kimono`, `green snake, grass snake`, `rubber eraser, rubber, pencil eraser`, `upright, upright piano`, `orange`, `revolver, six-gun, six-shooter`, `ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin`, `drum, membranophone, tympan`, `Dungeness crab, Cancer magister`, `lipstick, lip rouge`, `gong, tam-tam`, `fountain`, `tub, vat`, `malinois`, `sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita`, `German short-haired pointer`, `apron`, `Irish setter, red setter`, `dishrag, dishcloth`, `school bus`, `candle, taper, wax light`, `bib`, `cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM`, `power drill`, `English foxhound`, `miniskirt, mini`, `swing`, `slug`, `hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa`, `rifle`, `Saluki, gazelle hound`, `Sealyham terrier, Sealyham`, `bullet train, bullet`, `hyena, hyaena`, `ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus`, `toy terrier`, `goblet`, `safe`, `cup`, `electric guitar`, `red wine`, `restaurant, eating house, eating place, eatery`, `wall clock`, `washbasin, handbasin, washbowl, lavabo, wash-hand basin`, `red-breasted merganser, Mergus serrator`, `crate`, `banded gecko`, `hippopotamus, hippo, river horse, Hippopotamus amphibius`, `tick`, `tripod`, `sombrero`, `desk`, `sea slug, nudibranch`, `racer, race car, racing car`, `pizza, pizza pie`, `dining table, board`, `Saint Bernard, St Bernard`, `komondor`, `electric ray, crampfish, numbfish, torpedo`, `prairie chicken, prairie grouse, prairie fowl`, `coffee mug`, `hammer`, `golfcart, golf cart`, `unicycle, monocycle`, `bison`, `soup bowl`, `rapeseed`, `golden retriever`, `plastic bag`, `grey fox, gray fox, Urocyon cinereoargenteus`, `water tower`, `house finch, linnet, Carpodacus mexicanus`, `barbell`, `hair slide`, `tiger, Panthera tigris`, `black-footed ferret, ferret, Mustela nigripes`, `meat loaf, meatloaf`, `hand blower, blow dryer, blow drier, hair dryer, hair drier`, `overskirt`, `gibbon, Hylobates lar`, `Gila monster, Heloderma suspectum`, `toucan`, `snowmobile`, `pencil box, pencil case`, `scuba diver`, `cloak`, `Sussex spaniel`, `otter`, `Greater Swiss Mountain dog`, `great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias`, `torch`, `magpie`, `tiger shark, Galeocerdo cuvieri`, `wing`, `Border collie`, `bell cote, bell cot`, `sea anemone, anemone`, `teapot`, `sea urchin`, `screen, CRT screen`, `bookshop, bookstore, bookstall`, `oscilloscope, scope, cathode-ray oscilloscope, CRO`, `crib, cot`, `police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria`, `hartebeest`, `manhole cover`, `iPod`, `rock python, rock snake, Python sebae`, `nipple`, `suspension bridge`, `safety pin`, `sea lion`, `cougar, puma, catamount, mountain lion, painter, panther, Felis concolor`, `mantis, mantid`, `wardrobe, closet, press`, `projector`, `Granny Smith`, `diamondback, diamondback rattlesnake, Crotalus adamanteus`, `pirate, pirate ship`, `espresso maker`, `African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus`, `cradle`, `common newt, Triturus vulgaris`, `tricycle, trike, velocipede`, `bobsled, bobsleigh, bob`, `thunder snake, worm snake, Carphophis amoenus`, `thresher, thrasher, threshing machine`, `banjo`, `armadillo`, `pajama, pyjama, pj's, jammies`, `ski`, `Maltese dog, Maltese terrier, Maltese`, `leafhopper`, `book jacket, dust cover, dust jacket, dust wrapper`, `silky terrier, Sydney silky`, `Shih-Tzu`, `wallaby, brush kangaroo`, `cardigan`, `sturgeon`, `freight car`, `home theater, home theatre`, `sundial`, `African crocodile, Nile crocodile, Crocodylus niloticus`, `odometer, hodometer, mileometer, milometer`, `sliding door`, `vine snake`, `West Highland white terrier`, `mongoose`, `hornbill`, `beagle`, `European gallinule, Porphyrio porphyrio`, `submarine, pigboat, sub, U-boat`, `Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis`, `cock`, `pedestal, plinth, footstall`, `accordion, piano accordion, squeeze box`, `gown`, `lynx, catamount`, `guenon, guenon monkey`, `Walker hound, Walker foxhound`, `standard schnauzer`, `reel`, `hip, rose hip, rosehip`, `grasshopper, hopper`, `Dutch oven`, `stone wall`, `hard disc, hard disk, fixed disk`, `snow leopard, ounce, Panthera uncia`, `shopping cart`, `digital clock`, `hourglass`, `Border terrier`, `Old English sheepdog, bobtail`, `academic gown, academic robe, judge's robe`, `spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish`, `spotlight, spot`, `dome`, `barn spider, Araneus cavaticus`, `bee eater`, `basketball`, `cliff dwelling`, `folding chair`, `isopod`, `Doberman, Doberman pinscher`, `bittern`, `sunglasses, dark glasses, shades`, `picket fence, paling`, `Crock Pot`, `ibex, Capra ibex`, `neck brace`, `cardoon`, `cassette`, `amphibian, amphibious vehicle`, `minivan`, `analog clock`, `trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi`, `yurt`, `cliff, drop, drop-off`, `Bernese mountain dog`, `teddy, teddy bear`, `sloth bear, Melursus ursinus, Ursus ursinus`, `bassoon`, `toaster`, `ptarmigan`, `Gordon setter`, `night snake, Hypsiglena torquata`, `grand piano, grand`, `purse`, `clumber, clumber spaniel`, `shoji`, `hair spray`, `maillot`, `knee pad`, `space heater`, `bottlecap`, `chiffonier, commode`, `chain saw, chainsaw`, `sulphur butterfly, sulfur butterfly`, `pay-phone, pay-station`, `kelpie`, `mouse, computer mouse`, `car wheel`, `cornet, horn, trumpet, trump`, `container ship, containership, container vessel`, `matchstick`, `scabbard`, `American black bear, black bear, Ursus americanus, Euarctos americanus`, `langur`, `rock crab, Cancer irroratus`, `lionfish`, `speedboat`, `black stork, Ciconia nigra`, `knot`, `disk brake, disc brake`, `mosquito net`, `white stork, Ciconia ciconia`, `abacus`, `titi, titi monkey`, `grocery store, grocery, food market, market`, `waffle iron`, `pickelhaube`, `wooden spoon`, `Norwegian elkhound, elkhound`, `earthstar`, `sewing machine`, `balance beam, beam`, `potpie`, `chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour`, `Staffordshire bullterrier, Staffordshire bull terrier`, `switch, electric switch, electrical switch`, `dhole, Cuon alpinus`, `paddle, boat paddle`, `limousine, limo`, `Shetland sheepdog, Shetland sheep dog, Shetland`, `space bar`, `library`, `paddlewheel, paddle wheel`, `alligator lizard`, `Band Aid`, `Persian cat`, `bull mastiff`, `tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui`, `sports car, sport car`, `football helmet`, `laptop, laptop computer`, `lens cap, lens cover`, `tennis ball`, `violin, fiddle`, `lab coat, laboratory coat`, `cinema, movie theater, movie theatre, movie house, picture palace`, `weasel`, `bow tie, bow-tie, bowtie`, `macaw`, `dough`, `whiskey jug`, `microphone, mike`, `spoonbill`, `bassinet`, `mud turtle`, `velvet`, `warthog`, `plunger, plumber's helper`, `dugong, Dugong dugon`, `honeycomb`, `badger`, `dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk`, `bee`, `doormat, welcome mat`, `fountain pen`, `giant schnauzer`, `assault rifle, assault gun`, `limpkin, Aramus pictus`, `siamang, Hylobates syndactylus, Symphalangus syndactylus`, `albatross, mollymawk`, `confectionery, confectionary, candy store`, `harp`, `parachute, chute`, `barrel, cask`, `tank, army tank, armored combat vehicle, armoured combat vehicle`, `collie`, `kite`, `puck, hockey puck`, `stupa, tope`, `buckeye, horse chestnut, conker`, `patio, terrace`, `broom`, `Dandie Dinmont, Dandie Dinmont terrier`, `scorpion`, `agaric`, `balloon`, `bucket, pail`, `squirrel monkey, Saimiri sciureus`, `Eskimo dog, husky`, `zebra`, `garter snake, grass snake`, `indri, indris, Indri indri, Indri brevicaudatus`, `tractor`, `guinea pig, Cavia cobaya`, `maraca`, `red-backed sandpiper, dunlin, Erolia alpina`, `bullfrog, Rana catesbeiana`, `trilobite`, `Japanese spaniel`, `gorilla, Gorilla gorilla`, `monastery`, `centipede`, `terrapin`, `llama`, `long-horned beetle, longicorn, longicorn beetle`, `boxer`, `curly-coated retriever`, `mortar`, `hammerhead, hammerhead shark`, `goldfinch, Carduelis carduelis`, `garden spider, Aranea diademata`, `stopwatch, stop watch`, `grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus`, `leaf beetle, chrysomelid`, `birdhouse`, `king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica`, `stole`, `bell pepper`, `radiator`, `flatworm, platyhelminth`, `mushroom`, `Scotch terrier, Scottish terrier, Scottie`, `liner, ocean liner`, `toilet seat`, `lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens`, `zucchini, courgette`, `harvestman, daddy longlegs, Phalangium opilio`, `Newfoundland, Newfoundland dog`, `flamingo`, `whiptail, whiptail lizard`, `geyser`, `cleaver, meat cleaver, chopper`, `sea cucumber, holothurian`, `American egret, great white heron, Egretta albus`, `parking meter`, `beacon, lighthouse, beacon light, pharos`, `coucal`, `motor scooter, scooter`, `mitten`, `cannon`, `weevil`, `megalith, megalithic structure`, `stinkhorn, carrion fungus`, `ear, spike, capitulum`, `box turtle, box tortoise`, `snowplow, snowplough`, `tench, Tinca tinca`, `modem`, `tobacco shop, tobacconist shop, tobacconist`, `barn`, `skunk, polecat, wood pussy`, `African grey, African gray, Psittacus erithacus`, `Madagascar cat, ring-tailed lemur, Lemur catta`, `holster`, `barometer`, `sleeping bag`, `washer, automatic washer, washing machine`, `recreational vehicle, RV, R.V.`, `drake`, `tray`, `butcher shop, meat market`, `china cabinet, china closet`, `medicine chest, medicine cabinet`, `photocopier`, `Yorkshire terrier`, `starfish, sea star`, `racket, racquet`, `park bench`, `Labrador retriever`, `whistle`, `clog, geta, patten, sabot`, `volcano`, `quilt, comforter, comfort, puff`, `leopard, Panthera pardus`, `cauliflower`, `swimming trunks, bathing trunks`, `American chameleon, anole, Anolis carolinensis`, `alp`, `mortarboard`, `barracouta, snoek`, `cocker spaniel, English cocker spaniel, cocker`, `space shuttle`, `beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon`, `harmonica, mouth organ, harp, mouth harp`, `gasmask, respirator, gas helmet`, `wombat`, `Model T`, `wild boar, boar, Sus scrofa`, `hermit crab`, `flat-coated retriever`, `rotisserie`, `jinrikisha, ricksha, rickshaw`, `trifle`, `bannister, banister, balustrade, balusters, handrail`, `go-kart`, `bakery, bakeshop, bakehouse`, `ski mask`, `dock, dockage, docking facility`, `Egyptian cat`, `oxcart`, `redbone`, `shoe shop, shoe-shop, shoe store`, `convertible`, `ox`, `crayfish, crawfish, crawdad, crawdaddy`, `cowboy hat, ten-gallon hat`, `conch`, `spaghetti squash`, `toy poodle`, `saltshaker, salt shaker`, `microwave, microwave oven`, `triceratops`, `necklace`, `castle`, `streetcar, tram, tramcar, trolley, trolley car`, `eel`, `diaper, nappy, napkin`, `standard poodle`, `prayer rug, prayer mat`, `radio, wireless`, `crane`, `envelope`, `rule, ruler`, `gar, garfish, garpike, billfish, Lepisosteus osseus`, `spider monkey, Ateles geoffroyi`, `Irish wolfhound`, `German shepherd, German shepherd dog, German police dog, alsatian`, `umbrella`, `sunglass`, `aircraft carrier, carrier, flattop, attack aircraft carrier`, `water buffalo, water ox, Asiatic buffalo, Bubalus bubalis`, `jellyfish`, `groom, bridegroom`, `tree frog, tree-frog`, `steel arch bridge`, `lemon`, `pickup, pickup truck`, `vault`, `groenendael`, `baseball`, `junco, snowbird`, `maillot, tank suit`, `gazelle`, `jack-o'-lantern`, `military uniform`, `slide rule, slipstick`, `wire-haired fox terrier`, `acorn squash`, `electric fan, blower`, `Brittany spaniel`, `chimpanzee, chimp, Pan troglodytes`, `pillow`, `binder, ring-binder`, `schipperke`, `Afghan hound, Afghan`, `plate rack`, `car mirror`, `hand-held computer, hand-held microcomputer`, `papillon`, `schooner`, `Bedlington terrier`, `cellular telephone, cellular phone, cellphone, cell, mobile phone`, `altar`, `Chesapeake Bay retriever`, `cabbage butterfly`, `polecat, fitch, foulmart, foumart, Mustela putorius`, `comic book`, `French horn, horn`, `daisy`, `organ, pipe organ`, `mashed potato`, `acorn`, `fly`, `chain`, `American alligator, Alligator mississipiensis`, `mink`, `garbage truck, dustcart`, `totem pole`, `wine bottle`, `strawberry`, `cricket`, `European fire salamander, Salamandra salamandra`, `coral reef`, `Welsh springer spaniel`, `bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis`, `snorkel`, `bald eagle, American eagle, Haliaeetus leucocephalus`, `meerkat, mierkat`, `grille, radiator grille`, `nematode, nematode worm, roundworm`, `anemone fish`, `corn`, `loggerhead, loggerhead turtle, Caretta caretta`, `palace`, `suit, suit of clothes`, `pineapple, ananas`, `macaque`, `ping-pong ball`, `ram, tup`, `church, church building`, `koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus`, `hare`, `bath towel`, `strainer`, `yawl`, `otterhound, otter hound`, `table lamp`, `king snake, kingsnake`, `lotion`, `lion, king of beasts, Panthera leo`, `thatch, thatched roof`, `basset, basset hound`, `black and gold garden spider, Argiope aurantia`, `barber chair`, `proboscis monkey, Nasalis larvatus`, `consomme`, `Irish terrier`, `Irish water spaniel`, `common iguana, iguana, Iguana iguana`, `Weimaraner`, `Great Dane`, `pug, pug-dog`, `rhinoceros beetle`, `vase`, `brass, memorial tablet, plaque`, `kit fox, Vulpes macrotis`, `king penguin, Aptenodytes patagonica`, `vending machine`, `dalmatian, coach dog, carriage dog`, `rock beauty, Holocanthus tricolor`, `pole`, `cuirass`, `bolete`, `jackfruit, jak, jack`, `monarch, monarch butterfly, milkweed butterfly, Danaus plexippus`, `chow, chow chow`, `nail`, `packet`, `half track`, `Lhasa, Lhasa apso`, `boathouse`, `hay`, `valley, vale`, `slot, one-armed bandit`, `volleyball`, `carton`, `shower cap`, `tile roof`, `lacewing, lacewing fly`, `patas, hussar monkey, Erythrocebus patas`, `boa constrictor, Constrictor constrictor`, `black swan, Cygnus atratus`, `lampshade, lamp shade`, `mousetrap`, `crutch`, `vestment`, `Pekinese, Pekingese, Peke`, `tusker`, `warplane, military plane`, `sandal`, `screw`, `custard apple`, `Scottish deerhound, deerhound`, `spindle`, `keeshond`, `hummingbird`, `letter opener, paper knife, paperknife`, `cucumber, cuke`, `bearskin, busby, shako`, `fire engine, fire truck`, `trombone`, `ringneck snake, ring-necked snake, ring snake`, `sorrel`, `fire screen, fireguard`, `paper towel`, `flute, transverse flute`, `hotdog, hot dog, red hot`, `Indian elephant, Elephas maximus`, `mosque`, `stingray`, `Rhodesian ridgeback`, `four-poster` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lucky_model_en_4.1.0_3.0_1660166485109.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lucky_model_en_4.1.0_3.0_1660166485109.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_lucky_model", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_lucky_model", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_lucky_model| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|324.8 MB| --- layout: model title: English image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office ViTForImageClassification from mayoughi author: John Snow Labs name: image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office` is a English model originally trained by mayoughi. ## Predicted Entities `office`, `balcony`, `restaurant`, `hospital`, `inside apartment`, `airport`, `hallway`, `inside coffee house` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office_en_4.1.0_3.0_1660172866138.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office_en_4.1.0_3.0_1660172866138.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_apartment_office| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Google's Tapas Table Understanding (Medium, WTQ) author: John Snow Labs name: table_qa_tapas_medium_finetuned_wtq date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Medium Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_medium_finetuned_wtq_en_4.2.0_3.0_1664530490771.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_medium_finetuned_wtq_en_4.2.0_3.0_1664530490771.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_medium_finetuned_wtq","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_AraElectra_base_finetuned_ARCD","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_AraElectra_base_finetuned_ARCD","ar") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.squad_arcd.electra.base").predict("""ما هو اسمي؟|||"اسمي كلارا وأنا أعيش في بيركلي.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_AraElectra_base_finetuned_ARCD| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ar| |Size:|504.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/salti/AraElectra-base-finetuned-ARCD --- layout: model title: Legal Building And Public Works Document Classifier (EURLEX) author: John Snow Labs name: legclf_building_and_public_works_bert date: 2023-03-06 tags: [en, legal, classification, clauses, building_and_public_works, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_building_and_public_works_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Building_and_Public_Works or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Building_and_Public_Works`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_building_and_public_works_bert_en_1.0.0_3.0_1678111597578.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_building_and_public_works_bert_en_1.0.0_3.0_1678111597578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_building_and_public_works_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Building_and_Public_Works]| |[Other]| |[Other]| |[Building_and_Public_Works]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_building_and_public_works_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Building_and_Public_Works 0.85 0.85 0.85 33 Other 0.87 0.87 0.87 39 accuracy - - 0.86 72 macro-avg 0.86 0.86 0.86 72 weighted-avg 0.86 0.86 0.86 72 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from ThomasNLG) author: John Snow Labs name: t5_weighter_cnndm date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-weighter_cnndm-en` is a English model originally trained by `ThomasNLG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_weighter_cnndm_en_4.3.0_3.0_1675156764080.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_weighter_cnndm_en_4.3.0_3.0_1675156764080.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_weighter_cnndm","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_weighter_cnndm","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_weighter_cnndm| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|277.8 MB| ## References - https://huggingface.co/ThomasNLG/t5-weighter_cnndm-en - https://github.com/ThomasScialom/QuestEval - https://arxiv.org/abs/2103.12693 --- layout: model title: French CamemBert Embeddings (from joe8zhang) author: John Snow Labs name: camembert_embeddings_joe8zhang_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `joe8zhang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_joe8zhang_generic_model_fr_3.4.4_3.0_1653988899631.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_joe8zhang_generic_model_fr_3.4.4_3.0_1653988899631.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_joe8zhang_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/joe8zhang/dummy-model --- layout: model title: Korean ElectraForQuestionAnswering Small model (from monologg) Version-3 author: John Snow Labs name: electra_qa_small_v3_finetuned_korquad date: 2022-06-22 tags: [ko, open_source, electra, question_answering] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `koelectra-small-v3-finetuned-korquad` is a Korean model originally trained by `monologg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_small_v3_finetuned_korquad_ko_4.0.0_3.0_1655922310458.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_small_v3_finetuned_korquad_ko_4.0.0_3.0_1655922310458.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_small_v3_finetuned_korquad","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_small_v3_finetuned_korquad","ko") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.korquad.electra.small").predict("""내 이름은 무엇입니까?|||"제 이름은 클라라이고 저는 버클리에 살고 있습니다.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_small_v3_finetuned_korquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|53.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/monologg/koelectra-small-v3-finetuned-korquad --- layout: model title: Detect Drugs and Posology Entities (ner_posology_greedy) author: John Snow Labs name: ner_posology_greedy date: 2020-12-08 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.6.5 spark_version: 2.4 tags: [ner, licensed, clinical, en] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects drugs, dosage, form, frequency, duration, route, and drug strength in text. It differs from `ner_posology` in the sense that it chunks together drugs, dosage, form, strength, and route when they appear together, resulting in a bigger chunk. It is trained using `embeddings_clinical` so please use the same embeddings in the pipeline. ## Predicted Entities `DRUG`, `STRENGTH`, `DURATION`, `FREQUENCY`, `FORM`, `DOSAGE`, `ROUTE`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_POSOLOGY.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_greedy_en_2.6.4_2.4_1607422064676.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_greedy_en_2.6.4_2.4_1607422064676.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_posology_greedy", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day."]]).toDF("text")) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = NerDLModel.pretrained("ner_posology_greedy", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter)) val data = Seq("The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology.greedy").predict("""The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day.""") ```

## Results ```bash +----+----------------------------------+---------+-------+------------+ | | chunks | begin | end | entities | |---:|---------------------------------:|--------:|------:|-----------:| | 0 | 1 capsule of Advil 10 mg | 27 | 50 | DRUG | | 1 | magnesium hydroxide 100mg/1ml PO | 67 | 98 | DRUG | | 2 | for 5 days | 52 | 61 | DURATION | | 3 | 40 units of insulin glargine | 168 | 195 | DRUG | | 4 | at night | 197 | 204 | FREQUENCY | | 5 | 12 units of insulin lispro | 207 | 232 | DRUG | | 6 | with meals | 234 | 243 | FREQUENCY | | 7 | metformin 1000 mg | 250 | 266 | DRUG | | 8 | two times a day | 268 | 282 | FREQUENCY | +----+----------------------------------+---------+-------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_greedy| |Type:|ner| |Compatibility:|Spark NLP 2.6.5+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on augmented i2b2_med7 + FDA dataset with ``embeddings_clinical``, [https://www.i2b2.org/NLP/Medication](https://www.i2b2.org/NLP/Medication). ## Benchmarking | label | tp | fp | fn | prec | rec | f1 | |---------------|--------|-------|-------|------------|------------|------------| | B-DRUG | 29362 | 1679 | 1985 | 0.9459103 | 0.93667656 | 0.94127077 | | B-STRENGTH | 14018 | 1172 | 864 | 0.922844 | 0.9419433 | 0.9322958 | | I-DURATION | 6404 | 935 | 476 | 0.87259847 | 0.93081397 | 0.9007666 | | I-STRENGTH | 16686 | 1991 | 1292 | 0.8933983 | 0.9281344 | 0.9104351 | | I-FREQUENCY | 19743 | 1088 | 1081 | 0.9477702 | 0.94808877 | 0.9479294 | | B-FORM | 2733 | 526 | 780 | 0.8386008 | 0.7779676 | 0.80714715 | | B-DOSAGE | 2774 | 474 | 688 | 0.85406405 | 0.80127096 | 0.8268257 | | I-DOSAGE | 1357 | 490 | 844 | 0.7347049 | 0.6165379 | 0.67045456 | | I-DRUG | 37846 | 4103 | 3386 | 0.90219074 | 0.91787934 | 0.9099674 | | I-ROUTE | 208 | 30 | 62 | 0.8739496 | 0.77037036 | 0.8188976 | | B-ROUTE | 3061 | 340 | 451 | 0.9000294 | 0.87158316 | 0.88557786 | | B-DURATION | 2491 | 388 | 276 | 0.865231 | 0.900253 | 0.8823946 | | B-FREQUENCY | 13065 | 608 | 436 | 0.9555328 | 0.9677061 | 0.9615809 | | I-FORM | 154 | 69 | 386 | 0.69058293 | 0.2851852 | 0.40366974 | | | | | | | | | | Macro-average | 149902 | 13893 | 13007 | 0.8712434 | 0.82817215 | 0.849162 | | Micro-average | 149902 | 13893 | 13007 | 0.91518056 | 0.92015785 | 0.9176625 | --- layout: model title: Pipeline to Detect Risk Factors author: John Snow Labs name: ner_risk_factors_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_risk_factors](https://nlp.johnsnowlabs.com/2021/03/31/ner_risk_factors_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_risk_factors_pipeline_en_4.3.0_3.2_1678865692535.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_risk_factors_pipeline_en_4.3.0_3.2_1678865692535.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_risk_factors_pipeline", "en", "clinical/models") text = '''HISTORY OF PRESENT ILLNESS: The patient is a 40-year-old white male who presents with a chief complaint of "chest pain". The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. The severity of the pain has progressively increased. He describes the pain as a sharp and heavy pain which radiates to his neck & left arm. He ranks the pain a 7 on a scale of 1-10. He admits some shortness of breath & diaphoresis. He states that he has had nausea & 3 episodes of vomiting tonight. He denies any fever or chills. He admits prior episodes of similar pain prior to his PTCA in 1995. He states the pain is somewhat worse with walking and seems to be relieved with rest. There is no change in pain with positioning. He states that he took 3 nitroglycerin tablets sublingually over the past 1 hour, which he states has partially relieved his pain. The patient ranks his present pain a 4 on a scale of 1-10. The most recent episode of pain has lasted one-hour. The patient denies any history of recent surgery, head trauma, recent stroke, abnormal bleeding such as blood in urine or stool or nosebleed. REVIEW OF SYSTEMS: All other systems reviewed & are negative. PAST MEDICAL HISTORY: Diabetes mellitus type II, hypertension, coronary artery disease, atrial fibrillation, status post PTCA in 1995 by Dr. ABC. SOCIAL HISTORY: Denies alcohol or drugs. Smokes 2 packs of cigarettes per day. Works as a banker. FAMILY HISTORY: Positive for coronary artery disease (father & brother).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_risk_factors_pipeline", "en", "clinical/models") val text = "HISTORY OF PRESENT ILLNESS: The patient is a 40-year-old white male who presents with a chief complaint of "chest pain". The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. The severity of the pain has progressively increased. He describes the pain as a sharp and heavy pain which radiates to his neck & left arm. He ranks the pain a 7 on a scale of 1-10. He admits some shortness of breath & diaphoresis. He states that he has had nausea & 3 episodes of vomiting tonight. He denies any fever or chills. He admits prior episodes of similar pain prior to his PTCA in 1995. He states the pain is somewhat worse with walking and seems to be relieved with rest. There is no change in pain with positioning. He states that he took 3 nitroglycerin tablets sublingually over the past 1 hour, which he states has partially relieved his pain. The patient ranks his present pain a 4 on a scale of 1-10. The most recent episode of pain has lasted one-hour. The patient denies any history of recent surgery, head trauma, recent stroke, abnormal bleeding such as blood in urine or stool or nosebleed. REVIEW OF SYSTEMS: All other systems reviewed & are negative. PAST MEDICAL HISTORY: Diabetes mellitus type II, hypertension, coronary artery disease, atrial fibrillation, status post PTCA in 1995 by Dr. ABC. SOCIAL HISTORY: Denies alcohol or drugs. Smokes 2 packs of cigarettes per day. Works as a banker. FAMILY HISTORY: Positive for coronary artery disease (father & brother)." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.risk_factors.pipeline").predict("""HISTORY OF PRESENT ILLNESS: The patient is a 40-year-old white male who presents with a chief complaint of "chest pain". The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. The severity of the pain has progressively increased. He describes the pain as a sharp and heavy pain which radiates to his neck & left arm. He ranks the pain a 7 on a scale of 1-10. He admits some shortness of breath & diaphoresis. He states that he has had nausea & 3 episodes of vomiting tonight. He denies any fever or chills. He admits prior episodes of similar pain prior to his PTCA in 1995. He states the pain is somewhat worse with walking and seems to be relieved with rest. There is no change in pain with positioning. He states that he took 3 nitroglycerin tablets sublingually over the past 1 hour, which he states has partially relieved his pain. The patient ranks his present pain a 4 on a scale of 1-10. The most recent episode of pain has lasted one-hour. The patient denies any history of recent surgery, head trauma, recent stroke, abnormal bleeding such as blood in urine or stool or nosebleed. REVIEW OF SYSTEMS: All other systems reviewed & are negative. PAST MEDICAL HISTORY: Diabetes mellitus type II, hypertension, coronary artery disease, atrial fibrillation, status post PTCA in 1995 by Dr. ABC. SOCIAL HISTORY: Denies alcohol or drugs. Smokes 2 packs of cigarettes per day. Works as a banker. FAMILY HISTORY: Positive for coronary artery disease (father & brother).""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------------------------------|--------:|------:|:-------------|-------------:| | 0 | diabetic | 136 | 143 | DIABETES | 0.9992 | | 1 | coronary artery disease | 172 | 194 | CAD | 0.689667 | | 2 | Diabetes mellitus type II | 1315 | 1339 | DIABETES | 0.73075 | | 3 | hypertension | 1342 | 1353 | HYPERTENSION | 0.986 | | 4 | coronary artery disease | 1356 | 1378 | CAD | 0.882567 | | 5 | 1995 | 1422 | 1425 | PHI | 0.9999 | | 6 | ABC | 1434 | 1436 | PHI | 0.9999 | | 7 | Smokes 2 packs of cigarettes per day | 1481 | 1516 | SMOKER | 0.634257 | | 8 | banker | 1530 | 1535 | PHI | 0.9779 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_risk_factors_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Amharic Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_amharic date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, am, open_source] task: Named Entity Recognition language: am edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-swahili-finetuned-ner-amharic` is a Amharic model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_amharic_am_3.4.2_3.0_1652810385568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_amharic_am_3.4.2_3.0_1652810385568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_amharic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|am| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-swahili-finetuned-ner-amharic - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukm --- layout: model title: Word2Vec Embeddings in Sicilian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, scn, open_source] task: Embeddings language: scn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_scn_3.4.1_3.0_1647457121351.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_scn_3.4.1_3.0_1647457121351.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_tiny_finetuned_sqa","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_960h_lv60_self_4_gram', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_960h_lv60_self_4_gram", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_960h_lv60_self_4_gram| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Catalan RoBERTa embeddings author: cayorodriguez name: roberta_embeddings_bsc date: 2022-07-07 tags: [roberta, projecte_aina, ca, open_source] task: Embeddings language: ca edition: Spark NLP 3.4.4 spark_version: 3.0 supported: false recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Catalan Roberta Word Embeddings, used within the `PlanTL-GOB-ES/roberta-base-ca` project. This model requires a specific Tokenizer, as shown in the Python Examples section. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/cayorodriguez/roberta_embeddings_bsc_ca_3.4.4_3.0_1657198648319.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/cayorodriguez/roberta_embeddings_bsc_ca_3.4.4_3.0_1657198648319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") ex_list = ["aprox\.","pàg\.","p\.ex\.","gen\.","feb\.","abr\.","jul\.","set\.","oct\.","nov\.","dec\.","dr\.","dra\.","sr\.","sra\.","srta\.","núm\.","st\.","sta\.","pl\.","etc\.", "ex\."] ex_list_all = [] ex_list_all.extend(ex_list) ex_list_all.extend([x[0].upper() + x[1:] for x in ex_list]) ex_list_all.extend([x.upper() for x in ex_list]) tokenizer = Tokenizer() \ .setInputCols(['sentence']).setOutputCol('token')\ .setInfixPatterns(["(d|D)(els)","(d|D)(el)","(a|A)(ls)","(a|A)(l)","(p|P)(els)","(p|P)(el)",\ "([A-zÀ-ú_@]+)(-[A-zÀ-ú_@]+)",\ "(d'|D')([·A-zÀ-ú@_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+","(l'|L')([·A-zÀ-ú_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+", \ "(l'|l'|s'|s'|d'|d'|m'|m'|n'|n'|D'|D'|L'|L'|S'|S'|N'|N'|M'|M')([A-zÀ-ú_]+)",\ """([A-zÀ-ú·]+)(\.|,|\)|\?|!|;|\:|\"|”)(\.|,|\)|\?|!|;|\:|\"|”)+""",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|,|;|:|\?|,)+",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|')+([0-9A-zÀ-ú_]+)",\ "([0-9A-zÀ-ú·]+)(\.|\"|;|:|!|\?|$|$|”|“|'|,|%)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+([0-9]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+",\ "(d'|D'|l'|L')([·A-zÀ-ú@_]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)", \ "([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)"]) \ .setExceptions(ex_list_all).fit(data) embeddings = WordEmbeddingsModel.pretrained("roberta_embeddings_bsc","ca") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["M'encanta fer anar aixó."]]).toDF("text") result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bsc| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Community| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ca| |Size:|300.3 MB| |Case sensitive:|true| ## References projecte-aina/catalan_general_crawling @ huggingface --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10_en_4.3.0_3.0_1674214731317.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10_en_4.3.0_3.0_1674214731317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_256_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|427.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-10 --- layout: model title: Financial Executives Item Binary Classifier author: John Snow Labs name: finclf_executives_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `executives` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `executives` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_executives_item_en_1.0.0_3.2_1660154397820.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_executives_item_en_1.0.0_3.2_1660154397820.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_executives_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[executives]| |[other]| |[other]| |[executives]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_executives_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support executives 0.96 0.98 0.97 46 other 0.98 0.96 0.97 45 accuracy - - 0.97 91 macro-avg 0.97 0.97 0.97 91 weighted-avg 0.97 0.97 0.97 91 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_3_lr_3e_5_bs_32_ep_3 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-set_date_3-lr-3e-5-bs-32-ep-3` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_3_lr_3e_5_bs_32_ep_3_en_4.0.0_3.0_1657188664961.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_3_lr_3e_5_bs_32_ep_3_en_4.0.0_3.0_1657188664961.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_3_lr_3e_5_bs_32_ep_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_3_lr_3e_5_bs_32_ep_3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_3_lr_3e_5_bs_32_ep_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-set_date_3-lr-3e-5-bs-32-ep-3 --- layout: model title: Fast Neural Machine Translation Model from Albanian to English author: John Snow Labs name: opus_mt_sq_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, sq, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `sq` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_sq_en_xx_2.7.0_2.4_1609167620053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_sq_en_xx_2.7.0_2.4_1609167620053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_sq_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Assertion Status (assertion_dl_biobert_scope_L10R10) author: John Snow Labs name: assertion_dl_biobert_scope_L10R10 date: 2022-03-24 tags: [licensed, clinical, en, assertion, biobert] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 3.4.2 spark_version: 2.4 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained using `biobert_pubmed_base_cased` BERT token embeddings. It considers 10 tokens on the left and 10 tokens on the right side of the clinical entities extracted by NER models and assigns their assertion status based on their context in this scope. ## Predicted Entities `present`, `absent`, `possible`, `conditional`, `associated_with_someone_else`, `hypothetical` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_biobert_scope_L10R10_en_3.4.2_2.4_1648148217364.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_biobert_scope_L10R10_en_3.4.2_2.4_1648148217364.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") token = Tokenizer()\ .setInputCols(['sentence'])\ .setOutputCol('token') embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical_biobert", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("assertion_dl_biobert_scope_L10R10","en", "clinical/models") \ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[document, sentenceDetector, token, embeddings, clinical_ner, ner_converter, clinical_assertion]) text = "Patient with severe fever and sore throat. He shows no stomach pain and he maintained on an epidural and PCA for pain control. He also became short of breath with climbing a flight of stairs. After CT, lung tumor located at the right lower lobe. Father with Alzheimer." data = spark.createDataFrame([[text]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence","token","ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("assertion_dl_biobert_scope_L10R10","en", "clinical/models") .setInputCols(Array("sentence", "ner_chunk", "embeddings")) .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("Patient with severe fever and sore throat. He shows no stomach pain and he maintained on an epidural and PCA for pain control. He also became short of breath with climbing a flight of stairs. After CT, lung tumor located at the right lower lobe. Father with Alzheimer.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.assert.biobert_l10210").predict("""Patient with severe fever and sore throat. He shows no stomach pain and he maintained on an epidural and PCA for pain control. He also became short of breath with climbing a flight of stairs. After CT, lung tumor located at the right lower lobe. Father with Alzheimer.""") ```

## Results ```bash +---------------+---------+----------------------------+ |chunk |ner_label|assertion | +---------------+---------+----------------------------+ |severe fever |PROBLEM |present | |sore throat |PROBLEM |present | |stomach pain |PROBLEM |absent | |an epidural |TREATMENT|present | |PCA |TREATMENT|present | |pain control |TREATMENT|present | |short of breath|PROBLEM |conditional | |CT |TEST |present | |lung tumor |PROBLEM |present | |Alzheimer |PROBLEM |associated_with_someone_else| +---------------+---------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_dl_biobert_scope_L10R10| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|3.2 MB| ## References Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with `biobert_pubmed_base_cased`. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash label tp fp fn prec rec f1 absent 839 89 44 0.9040948 0.9501699 0.9265599 present 2436 127 168 0.9504487 0.9354839 0.9429069 conditional 29 21 24 0.58 0.5471698 0.5631067 associated_with_someone_else 39 11 6 0.78 0.8666670 0.8210527 hypothetical 164 44 11 0.7884616 0.9371429 0.8563969 possible 126 36 75 0.7777778 0.6268657 0.6942149 Macro-average 3633 328 328 0.7967971 0.8105832 0.8036310 Micro-average 3633 328 328 0.9171926 0.9171926 0.9171926 ``` --- layout: model title: Word2Vec Embeddings in Thai (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-10 tags: [cc, embeddings, fastText, word2vec, th, open_source] task: Embeddings language: th edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_th_3.4.1_3.0_1646924940460.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_th_3.4.1_3.0_1646924940460.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|th| |Size:|1.2 GB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Оcr base v2 for handwritten text author: John Snow Labs name: ocr_base_handwritten_v2 date: 2023-01-17 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 4.2.4 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Ocr base handwritten model v2 for recognise handwritten text based on TrOCR model pretrained on handwritten datasets. It is an Ocr base model for recognise handwritten text based on TrOcr architecture. The TrOCR model was proposed in TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. TrOCR consists of an image Transformer encoder and an autoregressive text Transformer decoder to perform optical character recognition (OCR). The abstract from the paper is the following: Text recognition is a long-standing research problem for document digitalization. Existing approaches for text recognition are usually built based on CNN for image understanding and RNN for char-level text generation. In addition, another language model is usually needed to improve the overall accuracy as a post-processing step. In this paper, we propose an end-to-end text recognition approach with pre-trained image Transformer and text Transformer models, namely TrOCR, which leverages the Transformer architecture for both image understanding and wordpiece-level text generation. The TrOCR model is simple but effective, and can be pre-trained with large-scale synthetic data and fine-tuned with human-labeled datasets. Experiments show that the TrOCR model outperforms the current state-of-the-art models on both printed and handwritten text recognition tasks. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/RECOGNIZE_HANDWRITTEN/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/jupyter/Cards/SparkOcrImageToTextHandwritten_V2.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_base_handwritten_v2_en_4.2.2_3.0_1670602309000.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_base_handwritten_v2_en_4.2.2_3.0_1670602309000.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) text_detector = ImageTextDetectorV2 \ .pretrained("image_text_detector_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("text_regions") \ .setWithRefiner(True) \ .setSizeThreshold(-1) \ .setLinkThreshold(0.3) \ .setWidth(500) ocr = ImageToTextV2Opt.pretrained("ocr_base_handwritten_v2", "en", "clinical/ocr") \ .setInputCols(["image", "text_regions"]) \ .setGroupImages(True) \ .setOutputCol("text") \ .setRegionsColumn("text_regions") draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("text_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.green) \ .setRotated(True) pipeline = PipelineModel(stages=[ binary_to_image, text_detector, ocr, draw_regions ]) # Download image: # !wget -q https://github.com/JohnSnowLabs/spark-ocr-workshop/raw/4.0.0-release-candidate/jupyter/data/handwritten/handwritten_example.jpg imagePath = 'handwritten_example.jpg' image_df = spark.read.format("binaryFile").load(imagePath) result = pipeline.transform(image_df).cache() ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val text_detector = ImageTextDetectorV2 .pretrained("image_text_detector_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("text_regions") .setWithRefiner(True) .setSizeThreshold(-1) .setLinkThreshold(0.3) .setWidth(500) val ocr = ImageToTextV2Opt .pretrained("ocr_base_handwritten_v2", "en", "clinical/ocr") .setInputCols(Array("image", "text_regions")) .setGroupImages(True) .setOutputCol("text") .setRegionsColumn("text_regions") val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("text_regions") .setOutputCol("image_with_regions") .setRectColor(Color.green) .setRotated(True) val pipeline = new PipelineModel().setStages(Array( binary_to_image, text_detector, ocr, draw_regions)) # Download image: # !wget -q https://github.com/JohnSnowLabs/spark-ocr-workshop/raw/4.0.0-release-candidate/jupyter/data/handwritten/handwritten_example.jpg val imagePath = "handwritten_example.jpg" val image_df = spark.read.format("binaryFile").load(imagePath) val result = pipeline.transform(image_df).cache() ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image3.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image3_out2.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash This is an example of handwritten beerxt Let's # check the performance ! I hope it will be awesome ``` ## Model Information {:.table-model} |---|---| |Model Name:|ocr_base_handwritten_v2| |Type:|ocr| |Compatibility:|Visual NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: English DistilBertForQuestionAnswering model (from ajaypyatha) author: John Snow Labs name: distilbert_qa_sdsqna date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sdsqna` is a English model originally trained by `ajaypyatha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sdsqna_en_4.0.0_3.0_1654728628088.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sdsqna_en_4.0.0_3.0_1654728628088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sdsqna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sdsqna","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.by_ajaypyatha").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sdsqna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ajaypyatha/sdsqna --- layout: model title: Pipeline to Detect Living Species(biobert_embeddings_biomedical) author: John Snow Labs name: ner_living_species_bert_pipeline date: 2023-03-13 tags: [pt, ner, clinical, licensed, biobert] task: Named Entity Recognition language: pt edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species_bert](https://nlp.johnsnowlabs.com/2022/06/22/ner_living_species_bert_pt_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_pipeline_pt_4.3.0_3.2_1678729438675.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_pipeline_pt_4.3.0_3.2_1678729438675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_bert_pipeline", "pt", "clinical/models") text = '''Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito..''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_bert_pipeline", "pt", "clinical/models") val text = "Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito.." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:------------|-------------:| | 0 | rapariga | 4 | 11 | HUMAN | 0.9849 | | 1 | pessoal | 41 | 47 | HUMAN | 0.9994 | | 2 | paciente | 182 | 189 | HUMAN | 1 | | 3 | gato | 368 | 371 | SPECIES | 0.9912 | | 4 | veterinário | 413 | 423 | HUMAN | 0.9909 | | 5 | Trichophyton rubrum | 632 | 650 | SPECIES | 0.9778 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|684.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal Agricultural Structures And Production Document Classifier (EURLEX) author: John Snow Labs name: legclf_agricultural_structures_and_production_bert date: 2023-03-06 tags: [en, legal, classification, clauses, agricultural_structures_and_production, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_agricultural_structures_and_production_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Agricultural_Structures_and_Production or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Agricultural_Structures_and_Production`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agricultural_structures_and_production_bert_en_1.0.0_3.0_1678111593399.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agricultural_structures_and_production_bert_en_1.0.0_3.0_1678111593399.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_agricultural_structures_and_production_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Agricultural_Structures_and_Production]| |[Other]| |[Other]| |[Agricultural_Structures_and_Production]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agricultural_structures_and_production_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.3 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Agricultural_Structures_and_Production 0.83 0.91 0.87 44 Other 0.90 0.81 0.85 43 accuracy - - 0.86 87 macro-avg 0.87 0.86 0.86 87 weighted-avg 0.87 0.86 0.86 87 ``` --- layout: model title: XLM-RoBERTa Base (xlm_roberta_base) author: John Snow Labs name: xlm_roberta_base date: 2021-05-25 tags: [xx, multilingual, embeddings, xlm_roberta, open_source] task: Embeddings language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [XLM-RoBERTa](https://ai.facebook.com/blog/-xlm-r-state-of-the-art-cross-lingual-understanding-through-self-supervision/) is a scaled cross-lingual sentence encoder. It is trained on 2.5T of data across 100 languages data filtered from Common Crawl. XLM-R achieves state-of-the-arts results on multiple cross-lingual benchmarks. The XLM-RoBERTa model was proposed in [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau, Kartikay Khandelwal, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco GuzmÃ¡n, Edouard Grave, Myle Ott, Luke Zettlemoyer, and Veselin Stoyanov. It is based on Facebook's RoBERTa model released in 2019. It is a large multi-lingual language model, trained on 2.5TB of filtered CommonCrawl data. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_xx_3.1.0_2.4_1621961851929.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_xx_3.1.0_2.4_1621961851929.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = XlmRoBertaEmbeddings.pretrained("xlm_roberta_base", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ``` ```scala val embeddings = XlmRoBertaEmbeddings.pretrained("xlm_roberta_base", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("xx.embed.xlm").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_base| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|xx| |Case sensitive:|true| ## Data Source [https://huggingface.co/xlm-roberta-base](https://huggingface.co/xlm-roberta-base) --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_name_all_904029569 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-name_all-904029569` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Name`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_all_904029569_en_4.3.1_3.0_1678783428948.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_all_904029569_en_4.3.1_3.0_1678783428948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_all_904029569","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_all_904029569","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_name_all_904029569| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-name_all-904029569 --- layout: model title: SDOH Tobacco Usage For Classification author: John Snow Labs name: genericclassifier_sdoh_tobacco_usage_sbiobert_cased_mli date: 2023-01-14 tags: [en, licensed, generic_classifier, sdoh, tobacco, clinical] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting tobacco use in clinical notes and trained by using GenericClassifierApproach annotator. `Present:` if the patient was a current consumer of tobacco. `Past:` the patient was a consumer in the past and had quit. `Never:` if the patient had never consumed tobacco. `None: ` if there was no related text. ## Predicted Entities `Present`, `Past`, `Never`, `None` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_TOBACCO/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_tobacco_usage_sbiobert_cased_mli_en_4.2.4_3.0_1673697468673.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_tobacco_usage_sbiobert_cased_mli_en_4.2.4_3.0_1673697468673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+---------+ | text| result| +----------------------------------------------------------------------------------------------------+---------+ |Retired schoolteacher, now substitutes. Lives with wife in location 1439. Has a 27 yo son and a 2...|[Present]| |The patient quit smoking approximately two years ago with an approximately a 40 pack year history...| [Past]| | The patient denies any history of smoking or alcohol abuse. She lives with her one daughter.| [Never]| |She was previously employed as a hairdresser, though says she hasnt worked in 4 years. Not report...| [None]| +----------------------------------------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_tobacco_usage_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| ## Benchmarking ```bash label precision recall f1-score support Never 0.89 0.90 0.90 487 None 0.86 0.78 0.82 269 Past 0.87 0.79 0.83 415 Present 0.63 0.82 0.71 203 accuracy - - 0.83 1374 macro-avg 0.81 0.82 0.81 1374 weighted-avg 0.84 0.83 0.83 1374 ``` --- layout: model title: Sentiment Analysis pipeline for English author: John Snow Labs name: analyze_sentiment date: 2021-03-24 tags: [open_source, english, analyze_sentiment, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The analyze_sentiment is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/analyze_sentiment_en_3.0.0_3.0_1616544471011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/analyze_sentiment_en_3.0.0_3.0_1616544471011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('analyze_sentiment', lang = 'en') result = pipeline.fullAnnotate("""Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("analyze_sentiment", lang = "en") val result = pipeline.fullAnnotate("""Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!""") ``` {:.nlu-block} ```python import nlu text = ["""Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!"""] result_df = nlu.load('en.classify').predict(text) result_df ```

## Results ```bash | | text | sentiment | |---:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------| | 0 | Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now! | positive | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|analyze_sentiment| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: Lemmatizer (Serbian, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, sr] task: Lemmatization language: sr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Serbian Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_sr_3.4.1_3.0_1646316491633.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_sr_3.4.1_3.0_1646316491633.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------+ |result | +---------------------+ |[Ниси, добар, од, ја]| +---------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|sr| |Size:|3.2 MB| --- layout: model title: Czech asr_wav2vec2_large_xlsr_czech TFWav2Vec2ForCTC from arampacha author: John Snow Labs name: asr_wav2vec2_large_xlsr_czech date: 2022-09-25 tags: [wav2vec2, cs, audio, open_source, asr] task: Automatic Speech Recognition language: cs edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_czech` is a Czech model originally trained by arampacha. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_czech_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_czech_cs_4.2.0_3.0_1664120388474.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_czech_cs_4.2.0_3.0_1664120388474.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_czech", "cs")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_czech", "cs") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_czech| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|cs| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from alex-apostolo) author: John Snow Labs name: roberta_qa_base_filtered_cuad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-filtered-cuad` is a English model originally trained by `alex-apostolo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_filtered_cuad_en_4.3.0_3.0_1674216293189.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_filtered_cuad_en_4.3.0_3.0_1674216293189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_filtered_cuad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_filtered_cuad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_filtered_cuad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|454.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/alex-apostolo/roberta-base-filtered-cuad --- layout: model title: Stopwords Remover for Czech language (358 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, cs, open_source] task: Stop Words Removal language: cs edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_cs_3.4.1_3.0_1646673248718.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_cs_3.4.1_3.0_1646673248718.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------+ |result | +---------------+ |[Nejste, lepší]| +---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|cs| |Size:|2.4 KB| --- layout: model title: English BertForQuestionAnswering Cased model (from irenelizihui) author: John Snow Labs name: bert_qa_irenelizihui_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `irenelizihui`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_irenelizihui_finetuned_squad_en_4.0.0_3.0_1657186550664.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_irenelizihui_finetuned_squad_en_4.0.0_3.0_1657186550664.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_irenelizihui_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_irenelizihui_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_irenelizihui_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/irenelizihui/bert-finetuned-squad --- layout: model title: English RoBERTa Embeddings (Sampling strategy 'full select') author: John Snow Labs name: roberta_embeddings_distilroberta_base_climate_f date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilroberta-base-climate-f` is a English model orginally trained by `climatebert`. Sampling strategy f: As expressed in the author's paper [here](https://arxiv.org/pdf/2110.12010.pdf), f is "full select" strategy, meaning all sentences from all corpora where selected. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_f_en_3.4.2_3.0_1649946254298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_f_en_3.4.2_3.0_1649946254298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_distilroberta_base_climate_f| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|310.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/climatebert/distilroberta-base-climate-f - https://arxiv.org/abs/2110.12010 --- layout: model title: Legal Multilabel Classification on Terms of Service (UNFAIR-ToS) author: John Snow Labs name: legmulticlf_unfair_tos date: 2023-03-08 tags: [en, legal, licensed, classification, unfair, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true recommended: true engine: tensorflow annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multilabel Text Classification model that can help you classify 8 types of unfair contractual terms (sentences), meaning terms that potentially violate user rights according to European consumer law. ## Predicted Entities `Arbitration`, `Choice_of_Law`, `Content_Removal`, `Contract_by_Using`, `Jurisdiction`, `Limitation_of_Liability`, `Unilateral_Change`, `Unilateral_Termination`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legmulticlf_unfair_tos_en_1.0.0_3.0_1678283272065.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legmulticlf_unfair_tos_en_1.0.0_3.0_1678283272065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['Unilateral_Change', 'Unilateral_Termination'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legmulticlf_unfair_tos| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|13.9 MB| ## References Train dataset available [here](https://github.com/coastalcph/lex-glue) ## Benchmarking ```bash label precision recall f1-score support Arbitration 1.00 0.82 0.90 11 Choice_of_Law 0.93 0.93 0.93 14 Content_Removal 0.80 0.57 0.67 21 Contract_by_Using 0.93 0.82 0.87 17 Jurisdiction 1.00 1.00 1.00 16 Limitation_of_Liability 0.81 0.80 0.81 60 Other 0.78 0.71 0.75 66 Unilateral_Change 0.94 0.84 0.89 38 Unilateral_Termination 0.78 0.81 0.79 36 micro-avg 0.85 0.79 0.82 279 macro-avg 0.89 0.81 0.85 279 weighted-avg 0.85 0.79 0.82 279 samples-avg 0.78 0.80 0.78 279 ``` --- layout: model title: ICD10CM Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-21 task: Entity Resolution edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/enterprise/healthcare/EntityResolution_ICD10_RxNorm_Detailed.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_clinical_en_2.4.5_2.4_1587491222166.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_clinical_en_2.4.5_2.4_1587491222166.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... icd10cm_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_clinical", "en", "clinical/models") \ .setInputCols(["ner_token", "chunk_embeddings"]) \ .setOutputCol("icd10cm_code") \ .setDistanceFunction("COSINE") \ .setNeighbours(5) pipeline_icd10cm = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, chunk_tokenizer, icd10cm_resolution]) pipeline_model = pipeline_icd10cm.fit(spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG."""]]).toDF("text")) result = pipeline_model.transform(data) ``` ```scala ... val icd10cm_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_clinical", "en", "clinical/models") .setInputCols("ner_token", "chunk_embeddings") .setOutputCol("icd10cm_code") .setDistanceFunction("COSINE") .setNeighbours(5) val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, chunk_tokenizer, icd10cm_resolution)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash | | chunk | entity | resolved_text | code | cms | |---|-----------------------------|-----------|----------------------------------------------------|--------|---------------------------------------------------| | 0 | T2DM), | PROBLEM | Type 2 diabetes mellitus with diabetic nephrop... | E1121 | Type 2 diabetes mellitus with diabetic nephrop... | | 1 | T2DM | PROBLEM | Type 2 diabetes mellitus with diabetic nephrop... | E1121 | Type 2 diabetes mellitus with diabetic nephrop... | | 2 | polydipsia | PROBLEM | Polydipsia | R631 | Polydipsia:::Anhedonia:::Galactorrhea | | 3 | interference from turbidity | PROBLEM | Non-working side interference | M2656 | Non-working side interference:::Hemoglobinuria... | | 4 | polyuria | PROBLEM | Other polyuria | R358 | Other polyuria:::Polydipsia:::Generalized edem... | | 5 | lipemia | PROBLEM | Glycosuria | R81 | Glycosuria:::Pure hyperglyceridemia:::Hyperchy... | | 6 | starvation ketosis | PROBLEM | Propionic acidemia | E71121 | Propionic acidemia:::Bartter's syndrome:::Hypo... | | 7 | HTG | PROBLEM | Pure hyperglyceridemia | E781 | Pure hyperglyceridemia:::Familial hypercholest... | ``` {:.model-param} ## Model Information {:.table-model} |----------------|-------------------------------| | Name: | chunkresolve_icd10cm_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.2+ | | License: | Licensed | |Edition:|Official| | |Input labels: | token, chunk_embeddings | |Output labels: | entity | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD10 Clinical Modification dataset with tenths of variations per code. https://www.icd10data.com/ICD10CM/Codes/ --- layout: model title: Detect Assertion Status (assertion_dl_healthcare) - supports confidence scores author: John Snow Labs name: assertion_dl_healthcare date: 2021-01-26 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.7.2 spark_version: 2.4 tags: [assertion, en, licensed, clinical, healthcare] supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Assign assertion status to clinical entities extracted by NER based on their context in the text. ## Predicted Entities `absent`, `present`, `conditional`, `associated_with_someone_else`, `hypothetical`, `possible`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_healthcare_en_2.7.2_2.4_1611646187271.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_healthcare_en_2.7.2_2.4_1611646187271.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------+---------+----------------------------+ |chunk |ner_label|assertion | +---------------+---------+----------------------------+ |severe fever |PROBLEM |present | |sore throat |PROBLEM |present | |stomach pain |PROBLEM |absent | |an epidural |TREATMENT|present | |PCA |TREATMENT|present | |pain control |TREATMENT|present | |short of breath|PROBLEM |conditional | |CT |TEST |present | |lung tumor |PROBLEM |present | |Alzheimer |PROBLEM |associated_with_someone_else| +---------------+---------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_dl_healthcare| |Compatibility:|Spark NLP 2.7.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| ## Data Source Trained on i2b2 assertion data ## Benchmarking ```bash label tp fp fn prec rec f1 absent 726 86 98 0.894089 0.881068 0.887531 present 2544 232 119 0.916427 0.955314 0.935466 conditional 18 13 37 0.580645 0.327273 0.418605 associated_with_someone_else 40 5 9 0.888889 0.816327 0.851064 hypothetical 132 13 26 0.910345 0.835443 0.871287 possible 96 45 105 0.680851 0.477612 0.561404 Macro-average 3556 394 394 0.811874 0.715506 0.76065 Micro-average 3556 394 394 0.900253 0.900253 0.900253 ``` --- layout: model title: Legal Compensation Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_compensation_agreement_bert date: 2023-01-29 tags: [en, legal, classification, compensation, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_compensation_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `compensation-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `compensation-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_compensation_agreement_bert_en_1.0.0_3.0_1674990338214.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_compensation_agreement_bert_en_1.0.0_3.0_1674990338214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_compensation_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[compensation-agreement]| |[other]| |[other]| |[compensation-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_compensation_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support compensation-agreement 1.00 0.95 0.97 40 other 0.96 1.00 0.98 55 accuracy - - 0.98 95 macro-avg 0.98 0.97 0.98 95 weighted-avg 0.98 0.98 0.98 95 ``` --- layout: model title: Legal Operating Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_operating_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, operating, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_operating_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `operating-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `operating-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_operating_agreement_bert_en_1.0.0_3.0_1669368532203.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_operating_agreement_bert_en_1.0.0_3.0_1669368532203.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_operating_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[operating-agreement]| |[other]| |[other]| |[operating-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_operating_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support operating-agreement 0.96 0.84 0.90 31 other 0.94 0.99 0.96 82 accuracy - - 0.95 113 macro-avg 0.95 0.91 0.93 113 weighted-avg 0.95 0.95 0.95 113 ``` --- layout: model title: Igbo Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_igbo date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, ig, open_source] task: Named Entity Recognition language: ig edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-swahili-finetuned-ner-igbo` is a Igbo model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_igbo_ig_3.4.2_3.0_1652809139166.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_igbo_ig_3.4.2_3.0_1652809139166.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_igbo| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ig| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-swahili-finetuned-ner-igbo - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman/ --- layout: model title: Translate Indic languages to English Pipeline author: John Snow Labs name: translate_inc_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, inc, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `inc` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_inc_en_xx_2.7.0_2.4_1609688074696.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_inc_en_xx_2.7.0_2.4_1609688074696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_inc_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_inc_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.inc.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_inc_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Kabyle asr_Kabyle_xlsr TFWav2Vec2ForCTC from Akashpb13 author: John Snow Labs name: pipeline_asr_Kabyle_xlsr date: 2022-09-24 tags: [wav2vec2, kab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: kab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Kabyle_xlsr` is a Kabyle model originally trained by Akashpb13. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Kabyle_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Kabyle_xlsr_kab_4.2.0_3.0_1664018945760.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Kabyle_xlsr_kab_4.2.0_3.0_1664018945760.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Kabyle_xlsr', lang = 'kab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Kabyle_xlsr", lang = "kab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Kabyle_xlsr| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|kab| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Finnish T5ForConditionalGeneration Mini Cased model (from Finnish-NLP) author: John Snow Labs name: t5_mini_nl8 date: 2023-01-31 tags: [fi, open_source, t5, tensorflow] task: Text Generation language: fi edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-mini-nl8-finnish` is a Finnish model originally trained by `Finnish-NLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_mini_nl8_fi_4.3.0_3.0_1675124948833.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_mini_nl8_fi_4.3.0_3.0_1675124948833.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_mini_nl8","fi") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_mini_nl8","fi") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_mini_nl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|fi| |Size:|315.9 MB| ## References - https://huggingface.co/Finnish-NLP/t5-mini-nl8-finnish - https://arxiv.org/abs/1910.10683 - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511 - https://arxiv.org/abs/2002.05202 - https://arxiv.org/abs/2109.10686 - http://urn.fi/urn:nbn:fi:lb-2017070501 - http://urn.fi/urn:nbn:fi:lb-2021050401 - http://urn.fi/urn:nbn:fi:lb-2018121001 - http://urn.fi/urn:nbn:fi:lb-2020021803 - https://sites.research.google/trc/about/ - https://github.com/google-research/t5x - https://github.com/spyysalo/yle-corpus - https://github.com/aajanki/eduskunta-vkk - https://sites.research.google/trc/ - https://www.linkedin.com/in/aapotanskanen/ - https://www.linkedin.com/in/rasmustoivanen/ --- layout: model title: Sentence Entity Resolver for RxNorm (sbert_jsl_medium_rxnorm_uncased embeddings) author: John Snow Labs name: sbertresolve_jsl_rxnorm_augmented_med date: 2021-12-28 tags: [clinical, entity_resolution, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbert_jsl_medium_rxnorm_uncased` Sentence Bert Embeddings. It is trained on the augmented version of the dataset which is used in previous RxNorm resolver models. Additionally, this model returns concept classes of the drugs in all_k_aux_labels column. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_jsl_rxnorm_augmented_med_en_3.3.4_2.4_1640686630389.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_jsl_rxnorm_augmented_med_en_3.3.4_2.4_1640686630389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained('sbert_jsl_medium_rxnorm_uncased', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_jsl_rxnorm_augmented_med", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver]) light_model = LightPipeline(rxnorm_pipelineModel) result = light_model.fullAnnotate(["Coumadin 5 mg", "aspirin", "Neurontin 300", "avandia 4 mg"]) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_rxnorm_uncased", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_jsl_rxnorm_augmented_med", "en", "clinical/models") \ .setInputCols(Array("sbert_embeddings")) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val light_model = LightPipeline(rxnorm_pipelineModel) val result = light_model.fullAnnotate(Array("Coumadin 5 mg", "aspirin", "Neurontin 300", "avandia 4 mg")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.augmented_med").predict("""Coumadin 5 mg""") ```

## Results ```bash | | RxNormCode | Resolution | all_k_results | all_k_distances | all_k_cosine_distances | all_k_resolutions | all_k_aux_labels | |---:|-------------:|:-------------------------------------------|:----------------------------------|:----------------------------------|:----------------------------------|:----------------------------------------------------------------|:----------------------------------| | 0 | 855333 | warfarin sodium 5 MG [Coumadin] | 855333:::855334:::1110792:::11... | 0.0000:::6.0548:::6.1667:::6.1... | 0.0000:::0.0515:::0.0536:::0.0... | warfarin sodium 5 MG [Coumadin]:::warfarin sodium 5 MG Oral ... | Branded Drug Comp:::Branded Dr... | | 1 | 1537020 | aspirin Effervescent Oral Tablet | 1537020:::1191:::202547:::1001... | 0.0000:::0.0000:::8.8123:::9.3... | 0.0000:::0.0000:::0.1145:::0.1... | aspirin Effervescent Oral Tablet:::aspirin:::Empirin:::Ecpir... | Clinical Drug Form:::Ingredien... | | 2 | 105029 | gabapentin 300 MG Oral Capsule [Neurontin] | 105029:::1718929:::1718930:::3... | 5.5969:::8.7502:::8.7502:::8.7... | 0.0452:::0.1092:::0.1092:::0.1... | gabapentin 300 MG Oral Capsule [Neurontin]:::olanzapine 300 ... | Branded Drug:::Clinical Drug C... | | 3 | 261242 | rosiglitazone 4 MG Oral Tablet [Avandia] | 261242:::2123140:::1792373:::8... | 0.0000:::7.1217:::7.7113:::8.4... | 0.0000:::0.0728:::0.0843:::0.1... | rosiglitazone 4 MG Oral Tablet [Avandia]:::erdafitinib 4 MG ... | Branded Drug:::Branded Drug Co... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_jsl_rxnorm_augmented_med| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|650.7 MB| |Case sensitive:|false| --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-16-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8_en_4.0.0_3.0_1657184626320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8_en_4.0.0_3.0_1657184626320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-16-finetuned-squad-seed-8 --- layout: model title: Catalan RobertaForTokenClassification Cased model (from softcatala) author: John Snow Labs name: roberta_token_classifier_fullstop_catalan_punctuation_prediction date: 2023-03-01 tags: [ca, open_source, roberta, token_classification, ner, tensorflow] task: Named Entity Recognition language: ca edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fullstop-catalan-punctuation-prediction` is a Catalan model originally trained by `softcatala`. ## Predicted Entities `.`, `?`, `-`, `:`, `,`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_fullstop_catalan_punctuation_prediction_ca_4.3.0_3.0_1677703587592.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_fullstop_catalan_punctuation_prediction_ca_4.3.0_3.0_1677703587592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_fullstop_catalan_punctuation_prediction","ca") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_fullstop_catalan_punctuation_prediction","ca") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_fullstop_catalan_punctuation_prediction| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ca| |Size:|457.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/softcatala/fullstop-catalan-punctuation-prediction - https://github.com/oliverguhr/fullstop-deep-punctuation-prediction --- layout: model title: Bemba (Zambia) asr_wav2vec2_large_xls_r_1b_bemba_fds TFWav2Vec2ForCTC from csikasote author: John Snow Labs name: asr_wav2vec2_large_xls_r_1b_bemba_fds date: 2022-09-24 tags: [wav2vec2, bem, audio, open_source, asr] task: Automatic Speech Recognition language: bem edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_1b_bemba_fds` is a Bemba (Zambia) model originally trained by csikasote. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_1b_bemba_fds_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_1b_bemba_fds_bem_4.2.0_3.0_1664043378039.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_1b_bemba_fds_bem_4.2.0_3.0_1664043378039.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_1b_bemba_fds", "bem")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_1b_bemba_fds", "bem") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_1b_bemba_fds| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|bem| |Size:|3.6 GB| --- layout: model title: Legal Conduct of business Clause Binary Classifier author: John Snow Labs name: legclf_conduct_of_business_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `conduct-of-business` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `conduct-of-business` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_conduct_of_business_clause_en_1.0.0_3.2_1660122262718.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_conduct_of_business_clause_en_1.0.0_3.2_1660122262718.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_conduct_of_business_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[conduct-of-business]| |[other]| |[other]| |[conduct-of-business]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_conduct_of_business_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support conduct-of-business 0.96 0.75 0.84 32 other 0.92 0.99 0.96 98 accuracy - - 0.93 130 macro-avg 0.94 0.87 0.90 130 weighted-avg 0.93 0.93 0.93 130 ``` --- layout: model title: Legal Subsidiaries Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_subsidiaries_bert date: 2023-03-05 tags: [en, legal, classification, clauses, subsidiaries, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Subsidiaries` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Subsidiaries`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subsidiaries_bert_en_1.0.0_3.0_1678050508907.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subsidiaries_bert_en_1.0.0_3.0_1678050508907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_subsidiaries_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Subsidiaries]| |[Other]| |[Other]| |[Subsidiaries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_subsidiaries_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.98 0.96 0.97 85 Subsidiaries 0.95 0.97 0.96 64 accuracy - - 0.97 149 macro-avg 0.97 0.97 0.97 149 weighted-avg 0.97 0.97 0.97 149 ``` --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_ff2000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-ff2000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff2000_en_4.3.0_3.0_1675123479854.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff2000_en_4.3.0_3.0_1675123479854.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_ff2000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_ff2000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_ff2000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|46.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-ff2000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Spanish BERT Sentence Base Cased Embedding author: John Snow Labs name: sent_bert_base_cased date: 2021-09-06 tags: [spanish, open_source, bert_sentence_embeddings, cased, es] task: Embeddings language: es edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BETO is a BERT model trained on a big Spanish corpus. BETO is of size similar to a BERT-Base and was trained with the Whole Word Masking technique. Below you find Tensorflow and Pytorch checkpoints for the uncased and cased versions, as well as some results for Spanish benchmarks comparing BETO with Multilingual BERT as well as other (not BERT-based) models. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_es_3.2.2_3.0_1630926259701.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_base_cased_es_3.2.2_3.0_1630926259701.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "es") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "es") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu nlu.load("es.embed_sentence.bert.base_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_base_cased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|es| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-128-finetuned-squad-seed-8` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8_en_4.0.0_3.0_1654191434774.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8_en_4.0.0_3.0_1654191434774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_128d_seed_8").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|381.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-128-finetuned-squad-seed-8 --- layout: model title: Korean DistilBertForQuestionAnswering Cased model (from pakupoko) author: John Snow Labs name: distilbert_qa_bizlin_model date: 2023-01-03 tags: [ko, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: ko edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bizlin-distil-model` is a Korean model originally trained by `pakupoko`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_bizlin_model_ko_4.3.0_3.0_1672765867213.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_bizlin_model_ko_4.3.0_3.0_1672765867213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bizlin_model","ko")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bizlin_model","ko") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_bizlin_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|104.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/pakupoko/bizlin-distil-model --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nh32 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nh32` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nh32_en_4.3.0_3.0_1675123694521.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nh32_en_4.3.0_3.0_1675123694521.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nh32","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nh32","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nh32| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|88.2 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nh32 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Typo Detector Pipeline for English author: ahmedlone127 name: distilbert_token_classifier_typo_detector_pipeline date: 2022-06-14 tags: [ner, bert, bert_for_token, typo, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [distilbert_token_classifier_typo_detector](https://nlp.johnsnowlabs.com/2022/01/19/distilbert_token_classifier_typo_detector_en.html). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TYPO_DETECTOR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/DistilBertForTokenClassification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/distilbert_token_classifier_typo_detector_pipeline_en_4.0.0_3.0_1655212406234.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/distilbert_token_classifier_typo_detector_pipeline_en_4.0.0_3.0_1655212406234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python typo_pipeline = PretrainedPipeline("distilbert_token_classifier_typo_detector_pipeline", lang = "en") typo_pipeline.annotate("He had also stgruggled with addiction during his tine in Congress.") ``` ```scala val typo_pipeline = new PretrainedPipeline("distilbert_token_classifier_typo_detector_pipeline", lang = "en") typo_pipeline.annotate("He had also stgruggled with addiction during his tine in Congress.") ```

## Results ```bash +----------+---------+ |chunk |ner_label| +----------+---------+ |stgruggled|PO | |tine |PO | +----------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_typo_detector_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|244.2 MB| ## Included Models - DocumentAssembler - TokenizerModel - DistilBertForTokenClassification - NerConverter - Finisher --- layout: model title: Legal Employment Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_employment_bert date: 2023-03-05 tags: [en, legal, classification, clauses, employment, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Employment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Employment`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_employment_bert_en_1.0.0_3.0_1678050529088.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_employment_bert_en_1.0.0_3.0_1678050529088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_employment_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Employment]| |[Other]| |[Other]| |[Employment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_employment_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Employment 0.96 0.96 0.96 27 Other 0.98 0.98 0.98 49 accuracy - - 0.97 76 macro-avg 0.97 0.97 0.97 76 weighted-avg 0.97 0.97 0.97 76 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from ponmari) author: John Snow Labs name: bert_qa_questionansweing date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `QuestionAnsweingBert` is a English model originally trained by `ponmari`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_questionansweing_en_4.0.0_3.0_1657182420944.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_questionansweing_en_4.0.0_3.0_1657182420944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_questionansweing","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_questionansweing","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_questionansweing| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ponmari/QuestionAnsweingBert --- layout: model title: English DistilBertForQuestionAnswering model (from Rocketknight1) author: John Snow Labs name: distilbert_qa_Rocketknight1_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Rocketknight1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Rocketknight1_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724414659.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Rocketknight1_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724414659.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Rocketknight1_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Rocketknight1_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Rocketknight1").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Rocketknight1_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Rocketknight1/distilbert-base-uncased-finetuned-squad --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_ff3000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-ff3000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff3000_en_4.3.0_3.0_1675123510286.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff3000_en_4.3.0_3.0_1675123510286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_ff3000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_ff3000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_ff3000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|62.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-ff3000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el16_dl8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el16-dl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el16_dl8_en_4.3.0_3.0_1675119813436.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el16_dl8_en_4.3.0_3.0_1675119813436.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el16_dl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el16_dl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el16_dl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|224.3 MB| ## References - https://huggingface.co/google/t5-efficient-small-el16-dl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English asr_wav2vec2_large_english TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_wav2vec2_large_english date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_english` is a English model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_english_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_english_en_4.2.0_3.0_1664020258828.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_english_en_4.2.0_3.0_1664020258828.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_english", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_english", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_english| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Pipeline to Detect Living Species author: John Snow Labs name: bert_token_classifier_ner_living_species_pipeline date: 2023-03-20 tags: [pt, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: pt edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_living_species](https://nlp.johnsnowlabs.com/2022/06/27/bert_token_classifier_ner_living_species_pt_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_pipeline_pt_4.3.0_3.2_1679304320046.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_pipeline_pt_4.3.0_3.2_1679304320046.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_living_species_pipeline", "pt", "clinical/models") text = '''Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_living_species_pipeline", "pt", "clinical/models") val text = "Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:------------|-------------:| | 0 | rapariga | 4 | 11 | HUMAN | 0.999888 | | 1 | pessoal | 41 | 47 | HUMAN | 0.99987 | | 2 | paciente | 182 | 189 | HUMAN | 0.999731 | | 3 | gato | 368 | 371 | SPECIES | 0.999365 | | 4 | veterinário | 413 | 423 | HUMAN | 0.982236 | | 5 | Trichophyton rubrum | 632 | 650 | SPECIES | 0.996602 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|666.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from English to Romanian author: John Snow Labs name: opus_mt_en_ro date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ro, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ro` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ro_xx_2.7.0_2.4_1609168722172.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ro_xx_2.7.0_2.4_1609168722172.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ro| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xls_r_300m_english_colab TFWav2Vec2ForCTC from shacharm author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_english_colab date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_english_colab` is a English model originally trained by shacharm. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_english_colab_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_english_colab_en_4.2.0_3.0_1664103544762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_english_colab_en_4.2.0_3.0_1664103544762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_english_colab', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_english_colab", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_english_colab| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Sentence Embeddings - sbert medium (tuned) author: John Snow Labs name: sbert_jsl_medium_umls_uncased date: 2021-06-30 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_umls_uncased_en_3.1.0_2.4_1625050119656.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_umls_uncased_en_3.1.0_2.4_1625050119656.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_umls_uncased","en","clinical/models").setInputCols(["sentence"]).setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_umls_uncased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_medium_umls_uncased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_medium_umls_uncased| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|false| ## Data Source Tuned on MedNLI and UMLS dataset ## Benchmarking ```bash MedNLI Acc: 0.744, STS (cos): 0.695 ``` ## Benchmarking ```bash MedNLI Score Acc 0.744 STS(cos) 0.695 ``` --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Russian (WikiNER 6B 100) author: John Snow Labs name: wikiner_6B_100 date: 2020-03-16 task: Named Entity Recognition language: ru edition: Spark NLP 2.4.4 spark_version: 2.4 tags: [ner, ru, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 6B 100 is trained with GloVe 6B 100 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_RU){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_RU.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_6B_100_ru_2.4.4_2.4_1584014001452.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_6B_100_ru_2.4.4_2.4_1584014001452.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_100d") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = NerDLModel.pretrained("wikiner_6B_100", "ru") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_100d") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("wikiner_6B_100", "ru") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла."""] ner_df = nlu.load('ru.ner.wikiner.glove.6B_100').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Уильям Генри Гейтс III|PER | |Microsoft |ORG | |За |ORG | |Microsoft Гейтс |MISC | |CEO |ORG | |Он |PER | |Гейтс |PER | |Сиэтле |LOC | |Вашингтон |LOC | |Полом Алленом |PER | |Альбукерке |LOC | |Нью-Мексико |LOC | |Microsoft |ORG | |Гейтс |PER | |Гейтс |PER | |Это |PER | |В июне 2006 |MISC | |Гейтс |PER | |Microsoft |ORG | |Фонде Билла |ORG | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_6B_100| |Type:|ner| |Compatibility:| Spark NLP 2.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ru| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://ru.wikipedia.org](https://ru.wikipedia.org) --- layout: model title: English BertForQuestionAnswering model (from KevinChoi) author: John Snow Labs name: bert_qa_KevinChoi_bert_finetuned_squad_accelerate date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model orginally trained by `KevinChoi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_KevinChoi_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535806391.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_KevinChoi_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535806391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_KevinChoi_bert_finetuned_squad_accelerate","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_KevinChoi_bert_finetuned_squad_accelerate","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.accelerate.by_KevinChoi").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_KevinChoi_bert_finetuned_squad_accelerate| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KevinChoi/bert-finetuned-squad-accelerate --- layout: model title: Explain Document ML Pipeline for English author: John Snow Labs name: explain_document_ml date: 2021-03-23 tags: [open_source, english, explain_document_ml, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_ml is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_ml_en_3.0.0_3.0_1616473253101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_ml_en_3.0.0_3.0_1616473253101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('explain_document_ml', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_ml", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.explain').predict(text) result_df ```

## Results ```bash | | document | sentence | token | spell | lemmas | stems | pos | |---:|:---------------------------------|:---------------------------------|:-------------------------------------------------|:------------------------------------------------|:------------------------------------------------|:-----------------------------------------------|:---------------------------------------| | 0 | ['Hello fronm John Snwow Labs!'] | ['Hello fronm John Snwow Labs!'] | ['Hello', 'fronm', 'John', 'Snwow', 'Labs', '!'] | ['Hello', 'front', 'John', 'Snow', 'Labs', '!'] | ['Hello', 'front', 'John', 'Snow', 'Labs', '!'] | ['hello', 'front', 'john', 'snow', 'lab', '!'] | ['UH', 'NN', 'NNP', 'NNP', 'NNP', '.'] || | document | sentence | token | spell | lemmas | stems | pos | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_ml| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: English asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4 TFWav2Vec2ForCTC from chrisvinsen author: John Snow Labs name: asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4` is a English model originally trained by chrisvinsen. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_en_4.2.0_3.0_1664103661065.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_en_4.2.0_3.0_1664103661065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Legal Salary Clause Binary Classifier author: John Snow Labs name: legclf_salary_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `salary` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `salary` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_salary_clause_en_1.0.0_3.2_1660123961758.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_salary_clause_en_1.0.0_3.2_1660123961758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_salary_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[salary]| |[other]| |[other]| |[salary]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_salary_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 79 salary 1.00 1.00 1.00 33 accuracy - - 1.00 112 macro-avg 1.00 1.00 1.00 112 weighted-avg 1.00 1.00 1.00 112 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Lingala author: John Snow Labs name: opus_mt_en_ln date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ln, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ln` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ln_xx_2.7.0_2.4_1609170351544.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ln_xx_2.7.0_2.4_1609170351544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ln| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_100h_with_lm_turkish TFWav2Vec2ForCTC from gorkemgoknar author: John Snow Labs name: asr_wav2vec2_base_100h_with_lm_turkish date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_with_lm_turkish` is a English model originally trained by gorkemgoknar. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_100h_with_lm_turkish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_with_lm_turkish_en_4.2.0_3.0_1664038528506.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_with_lm_turkish_en_4.2.0_3.0_1664038528506.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_100h_with_lm_turkish", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_100h_with_lm_turkish", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_100h_with_lm_turkish| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.3 MB| --- layout: model title: Detect Diseases in Medical Text author: John Snow Labs name: bert_token_classifier_ner_bc5cdr_disease date: 2022-07-25 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Chemicals, diseases, and their relations are among the most searched topics by PubMed users worldwide as they play central roles in many areas of biomedical research and healthcare, such as drug discovery and safety surveillance. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. The model detects disease from a medical text ## Predicted Entities `DISEASE` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc5cdr_disease_en_4.0.0_3.0_1658754395259.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc5cdr_disease_en_4.0.0_3.0_1658754395259.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------+-------+ |ner_chunk |label | +---------------------+-------+ |interstitial cystitis|DISEASE| |mastocytosis |DISEASE| |cystitis |DISEASE| |prostate cancer |DISEASE| +---------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bc5cdr_disease| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://github.com/cambridgeltl/MTL-Bioinformatics-2016](https://github.com/cambridgeltl/MTL-Bioinformatics-2016) ## Benchmarking ```bash label precision recall f1-score support B-DISEASE 0.7905 0.9146 0.8480 4424 I-DISEASE 0.6521 0.8725 0.7464 2737 micro-avg 0.7328 0.8985 0.8072 7161 macro-avg 0.7213 0.8935 0.7972 7161 weighted-avg 0.7376 0.8985 0.8092 7161 ``` --- layout: model title: Urdu DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_ur_cased date: 2022-04-12 tags: [distilbert, embeddings, ur, open_source] task: Embeddings language: ur edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-ur-cased` is a Urdu model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_ur_cased_ur_3.4.2_3.0_1649783731492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_ur_cased_ur_3.4.2_3.0_1649783731492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_ur_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ur| |Size:|186.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-ur-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English BertForQuestionAnswering model (from ixa-ehu) author: John Snow Labs name: bert_qa_SciBERT_SQuAD_QuAC date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `SciBERT-SQuAD-QuAC` is a English model orginally trained by `ixa-ehu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_SciBERT_SQuAD_QuAC_en_4.0.0_3.0_1654179044906.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_SciBERT_SQuAD_QuAC_en_4.0.0_3.0_1654179044906.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_SciBERT_SQuAD_QuAC","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_SciBERT_SQuAD_QuAC","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.scibert.by_ixa-ehu").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_SciBERT_SQuAD_QuAC| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ixa-ehu/SciBERT-SQuAD-QuAC - https://www.aclweb.org/anthology/P18-2124/ - https://arxiv.org/abs/1808.07036 --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886_de_4.2.0_3.0_1664115914159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886_de_4.2.0_3.0_1664115914159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s886| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Named Entity Recognition in Romanian Official Documents (Medium) author: John Snow Labs name: legner_romanian_official_md date: 2022-11-10 tags: [ro, ner, legal, licensed] task: Named Entity Recognition language: ro edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a medium version of NER model that extracts PER(Person), LOC(Location), ORG(Organization), DATE and LEGAL entities from Romanian Official Documents. Different from small version, it labels all entities related to legal domain as LEGAL. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE`, `LEGAL` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/LEGNER_ROMANIAN_OFFICIAL/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_md_ro_1.0.0_3.0_1668083301892.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_md_ro_1.0.0_3.0_1668083301892.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence")\ tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_base_cased", "ro")\ .setInputCols("sentence", "token")\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_romanian_official_md", "ro", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""Anexa nr. 1 la Ordinul ministrului sănătății nr. 1.468 / 2018 pentru aprobarea prețurilor maximale ale medicamentelor de uz uman, valabile în România, care pot fi utilizate / comercializate de către deținătorii de autorizație de punere pe piață a medicamentelor sau reprezentanții acestora, distribuitorii angro și furnizorii de servicii medicale și medicamente pentru acele medicamente care fac obiectul unei relații contractuale cu Ministerul Sănătății, casele de asigurări de sănătate și / sau direcțiile de sănătate publică județene și a municipiului București, cuprinse în Catalogul național al prețurilor medicamentelor autorizate de punere pe piață în România, a prețurilor de referință generice și a prețurilor de referință inovative, publicat în Monitorul Oficial al României, Partea I nr. 989 și 989 bis din 22 noiembrie 2018, cu modificările și completările ulterioare, se modifică și se completează conform anexei care face parte integrantă din prezentul ordin."""]]).toDF("text") result = model.transform(data) ```

## Results ```bash +----------------------------------------------+-----+ |chunk |label| +----------------------------------------------+-----+ |Ordinul ministrului sănătății nr. 1.468 / 2018|LEGAL| |România |LOC | |Ministerul Sănătății |ORG | |București |LOC | |România |LOC | |Monitorul Oficial al României |ORG | |22 noiembrie 2018 |DATE | +----------------------------------------------+-----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_romanian_official_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.5 MB| ## References Dataset is available [here](https://zenodo.org/record/7025333#.Y2zsquxBx83). ## Benchmarking ```bash label precision recall f1-score support DATE 0.84 0.92 0.88 218 LEGAL 0.89 0.96 0.92 337 LOC 0.82 0.77 0.79 158 ORG 0.87 0.88 0.88 463 PER 0.97 0.97 0.97 87 micro-avg 0.87 0.90 0.89 1263 macro-avg 0.88 0.90 0.89 1263 weighted-avg 0.87 0.90 0.89 1263 ``` --- layout: model title: German Electra Embeddings (from deepset) author: John Snow Labs name: electra_embeddings_gelectra_large_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gelectra-large-generator` is a German model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_gelectra_large_generator_de_3.4.4_3.0_1652786854236.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_gelectra_large_generator_de_3.4.4_3.0_1652786854236.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_gelectra_large_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|194.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gelectra-large-generator - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Legal Deterioration Of The Environment Document Classifier (EURLEX) author: John Snow Labs name: legclf_deterioration_of_the_environment_bert date: 2023-03-06 tags: [en, legal, classification, clauses, deterioration_of_the_environment, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_deterioration_of_the_environment_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Deterioration_of_The_Environment or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Deterioration_of_The_Environment`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_deterioration_of_the_environment_bert_en_1.0.0_3.0_1678111773583.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_deterioration_of_the_environment_bert_en_1.0.0_3.0_1678111773583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_deterioration_of_the_environment_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Deterioration_of_The_Environment]| |[Other]| |[Other]| |[Deterioration_of_The_Environment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_deterioration_of_the_environment_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Deterioration_of_The_Environment 0.92 0.91 0.91 196 Other 0.91 0.92 0.91 191 accuracy - - 0.91 387 macro-avg 0.91 0.91 0.91 387 weighted-avg 0.91 0.91 0.91 387 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from pakupoko) author: John Snow Labs name: distilbert_qa_bizlin_distil_model date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bizlin-distil-model` is a English model originally trained by `pakupoko`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_bizlin_distil_model_en_4.0.0_3.0_1654723375668.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_bizlin_distil_model_en_4.0.0_3.0_1654723375668.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bizlin_distil_model","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bizlin_distil_model","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.by_pakupoko").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_bizlin_distil_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|104.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/pakupoko/bizlin-distil-model --- layout: model title: English RobertaForQuestionAnswering (from deepset) author: John Snow Labs name: roberta_qa_tinyroberta_squad2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tinyroberta-squad2` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_tinyroberta_squad2_en_4.0.0_3.0_1655740021196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_tinyroberta_squad2_en_4.0.0_3.0_1655740021196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_tinyroberta_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_tinyroberta_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.tiny.by_deepset").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_tinyroberta_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|307.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/tinyroberta-squad2 - https://www.linkedin.com/company/deepset-ai/ - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://arxiv.org/pdf/1909.10351.pdf - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack - http://deepset.ai/ - https://haystack.deepset.ai/ - http://www.deepset.ai/jobs - https://twitter.com/deepset_ai - https://github.com/deepset-ai/haystack/discussions - https://github.com/deepset-ai/haystack/ - https://deepset.ai - https://deepset.ai/germanquad - https://haystack.deepset.ai - https://deepset.ai/german-bert --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-6` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6_en_4.0.0_3.0_1654537973401.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6_en_4.0.0_3.0_1654537973401.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_1024d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|390.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-6 --- layout: model title: English DistilBertForQuestionAnswering model (from adamlin) author: John Snow Labs name: distilbert_qa_base_cased_sgd_qa_step5000 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-sgd_qa-step5000` is a English model originally trained by `adamlin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_sgd_qa_step5000_en_4.0.0_3.0_1654723671592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_sgd_qa_step5000_en_4.0.0_3.0_1654723671592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_sgd_qa_step5000","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_sgd_qa_step5000","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_cased.by_adamlin").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_sgd_qa_step5000| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/adamlin/distilbert-base-cased-sgd_qa-step5000 --- layout: model title: Sentiment Analysis for Urdu (IMDB Review dataset) author: John Snow Labs name: sentimentdl_urduvec_imdb date: 2021-01-09 task: Sentiment Analysis language: ur edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [ur, open_source, sentiment] supported: true annotator: SentimentDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Analyse sentiment in reviews by classifying them as `positive` or `negative`. This model is trained using `urduvec_140M_300d` word embeddings. The word embeddings are then converted to sentence embeddings before feeding to the sentiment classifier which uses a DL architecture to classify sentences. ## Predicted Entities `positive` , `negative` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentimentdl_urduvec_imdb_ur_2.7.1_2.4_1610185467237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentimentdl_urduvec_imdb_ur_2.7.1_2.4_1610185467237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, SentenceEmbeddings.

## Results ```bash | | document | sentiment | |---:|---------------------------------------------------------------------------------------------------------:|--------------:| | 0 |مجھے واقعی یہ شو سند ہے۔ یہی وجہ ہے کہ مجھے حال ہی میں یہ جان کر مایوسی ہوئی ہے کہ جارج لوپیز ایک | positive | | 1 |بالکل بھی اچھ ،ی کام نہیں کیا گیا ، پوری فلم صرف گرڈج تھی اور کہیں بھی بے ترتیب لوگوں کو ہلاک نہیں | negative | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentimentdl_urduvec_imdb| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[sentiment]| |Language:|ur| |Dependencies:|urduvec_140M_300d| ## Data Source This models in trained using data from https://www.kaggle.com/akkefa/imdb-dataset-of-50k-movie-translated-urdu-reviews ## Benchmarking ```bash loss: 2428.622 - acc: 0.8181 - val_acc: 80.0 ``` --- layout: model title: Arabic BertForMaskedLM Base Cased model (from CAMeL-Lab) author: John Snow Labs name: bert_embeddings_base_arabic_camel_mix date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabic-camelbert-mix` is a Arabic model originally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_mix_ar_4.2.4_3.0_1670015990923.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_mix_ar_4.2.4_3.0_1670015990923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_mix","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_mix","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabic_camel_mix| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-mix - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://arxiv.org/abs/2103.06678 - https://zenodo.org/record/3891466#.YEX4-F0zbzc - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: Split Sentences in Healthcare Texts author: John Snow Labs name: sentence_detector_dl_healthcare class: DeepSentenceDetector language: en nav_key: models repository: clinical/models date: 2020-09-13 task: Sentence Detection edition: Healthcare NLP 2.6.0 spark_version: 2.4 tags: [clinical,sentence_detection,en] supported: true annotator: SentenceDetectorDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SENTENCE_DETECTOR_HC/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_2.6.0_2.4_1600001082565.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_2.6.0_2.4_1600001082565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl_healthcare","en","clinical/models") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""John loves Mary.Mary loves Peter. Peter loves Helen .Helen loves John; Total: four people involved.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","en","clinical/models") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("John loves Mary.Mary loves Peter. Peter loves Helen .Helen loves John; Total: four people involved.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.detect_sentence.clinical").predict("""John loves Mary.Mary loves Peter. Peter loves Helen .Helen loves John; Total: four people involved.""") ```

{:.h2_title} ## Results ```bash +---+------------------------------+ | 0 | John loves Mary. | +---+------------------------------+ | 1 | Mary loves Peter | +---+------------------------------+ | 2 | Peter loves Helen . | +---+------------------------------+ | 3 | Helen loves John; | +---+------------------------------+ | 4 | Total: four people involved. | +---+------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|-------------------------------------------| | Name: | sentence_detector_dl_healthcare | | Type: | DeepSentenceDetector | | Compatibility: | Spark NLP 2.6.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document] | |Output labels: | sentence | | Language: | en | {:.h2_title} ## Data Source Healthcare SDDL model is trained on domain (healthcare) specific text, annotated internally, to generalize further on clinical notes. {:.h2_title} ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Finnish Lemmatizer author: John Snow Labs name: lemma date: 2020-05-05 12:35:00 +0800 task: Lemmatization language: fi edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, fi] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_fi_2.5.0_2.4_1588671290521.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_fi_2.5.0_2.4_1588671290521.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "fi") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "fi") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä."""] lemma_df = nlu.load('fi.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=2, result='se', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=4, end=10, result='lisäksi', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=11, end=11, result=',', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=13, end=16, result='että', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=18, end=20, result='hän', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|fi| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Google T5 (Text-To-Text Transfer Transformer) Small author: John Snow Labs name: t5_small date: 2021-01-08 task: [Question Answering, Summarization, Translation] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, t5, summarization, translation, en, seq2seq] supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The T5 transformer model described in the seminal paper "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer". This model can perform a variety of tasks, such as text summarization, question answering, and translation. More details about using the model can be found in the paper (https://arxiv.org/pdf/1910.10683.pdf). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5TRANSFORMER/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_en_2.7.1_2.4_1610133219885.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_en_2.7.1_2.4_1610133219885.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Either set the following tasks or have them inline with your input: - summarize: - translate English to German: - translate English to French: - stsb sentence1: Big news. sentence2: No idea. The full list of tasks is in the Appendix of the paper: https://arxiv.org/pdf/1910.10683.pdf

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[t5]| |Language:|en| ## Data Source https://huggingface.co/t5-small --- layout: model title: Translate English to North Germanic languages Pipeline author: John Snow Labs name: translate_en_gmq date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, gmq, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `gmq` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_gmq_xx_2.7.0_2.4_1609698966439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_gmq_xx_2.7.0_2.4_1609698966439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_gmq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_gmq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.gmq').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_gmq| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from gdario) author: John Snow Labs name: bert_qa_biobert_bioasq date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert_bioasq` is a English model orginally trained by `gdario`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_bioasq_en_4.0.0_3.0_1654185669067.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_bioasq_en_4.0.0_3.0_1654185669067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_bioasq","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_bioasq","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.biobert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_bioasq| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/gdario/biobert_bioasq --- layout: model title: English asr_wav2vec2_xls_r_300m_kh TFWav2Vec2ForCTC from kongkeaouch author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_kh date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_kh` is a English model originally trained by kongkeaouch. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_kh_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_kh_en_4.2.0_3.0_1664025155355.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_kh_en_4.2.0_3.0_1664025155355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_kh', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_kh", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_kh| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Pledge Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_pledge_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, pledge, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_pledge_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `pledge-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `pledge-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_pledge_agreement_bert_en_1.0.0_3.0_1670349668991.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_pledge_agreement_bert_en_1.0.0_3.0_1670349668991.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_pledge_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[pledge-agreement]| |[other]| |[other]| |[pledge-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_pledge_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.98 0.97 65 pledge-agreement 0.96 0.88 0.92 26 accuracy - - 0.96 91 macro-avg 0.96 0.93 0.94 91 weighted-avg 0.96 0.96 0.96 91 ``` --- layout: model title: Chinese Bert Embeddings (Roberta, Whole Word Masking) author: John Snow Labs name: bert_embeddings_chinese_roberta_wwm_ext date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-roberta-wwm-ext` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_ext_zh_3.4.2_3.0_1649668840010.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_ext_zh_3.4.2_3.0_1649668840010.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_wwm_ext| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-roberta-wwm-ext - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English asr_model_2 TFWav2Vec2ForCTC from niclas author: John Snow Labs name: pipeline_asr_model_2 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_2` is a English model originally trained by niclas. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_model_2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_2_en_4.2.0_3.0_1664097773728.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_2_en_4.2.0_3.0_1664097773728.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_model_2', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_model_2", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_model_2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_el16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-el16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el16_en_4.3.0_3.0_1675110985082.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el16_en_4.3.0_3.0_1675110985082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_el16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_el16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_el16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|529.5 MB| ## References - https://huggingface.co/google/t5-efficient-base-el16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Financing And Investment Document Classifier (EURLEX) author: John Snow Labs name: legclf_financing_and_investment_bert date: 2023-03-06 tags: [en, legal, classification, clauses, financing_and_investment, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_financing_and_investment_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Financing_and_Investment or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Financing_and_Investment`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_financing_and_investment_bert_en_1.0.0_3.0_1678111700202.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_financing_and_investment_bert_en_1.0.0_3.0_1678111700202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_financing_and_investment_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Financing_and_Investment]| |[Other]| |[Other]| |[Financing_and_Investment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_financing_and_investment_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Financing_and_Investment 0.84 0.91 0.87 45 Other 0.91 0.84 0.87 50 accuracy - - 0.87 95 macro-avg 0.87 0.88 0.87 95 weighted-avg 0.88 0.87 0.87 95 ``` --- layout: model title: Javanese DistilBERT Embeddings (Small, Imdb) author: John Snow Labs name: distilbert_embeddings_javanese_distilbert_small_imdb date: 2022-04-12 tags: [distilbert, embeddings, jv, open_source] task: Embeddings language: jv edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `javanese-distilbert-small-imdb` is a Javanese model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_distilbert_small_imdb_jv_3.4.2_3.0_1649783783892.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_distilbert_small_imdb_jv_3.4.2_3.0_1649783783892.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_javanese_distilbert_small_imdb| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|248.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-distilbert-small-imdb - https://arxiv.org/abs/1910.01108 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: Translate Shona to English Pipeline author: John Snow Labs name: translate_sn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, sn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `sn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_sn_en_xx_2.7.0_2.4_1609688774813.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_sn_en_xx_2.7.0_2.4_1609688774813.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_sn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_sn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.sn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_sn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for RxNorm According to National Institute of Health (NIH) Database (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm_nih date: 2023-02-22 tags: [entity_resolution, rxnorm, clinical, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 4.3.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes according to the National Institute of Health (NIH) database using `sbiobert_base_cased_mli` Sentence Bert Embeddings. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_nih_en_4.3.0_3.0_1677106956679.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_nih_en_4.3.0_3.0_1677106956679.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | sent_id | ner_chunk | entity | rxnorm_code | all_codes | resolutions | |---:|----------:|:---------------------|:---------|--------------:|:------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| | 0 | 0 | folic acid 1 mg | DRUG | 12281181 | ['12281181', '12283696', '12270292', '12306595', 1227889...| ['folic acid 1 MG [folic acid 1 MG]', 'folic acid 1.1 MG [folic acid 1.1 MG]', 'folic acid 1 MG/ML [folic acid 1 MG/ML]', 'folic a...| | 1 | 0 | levothyroxine 0.1 mg | DRUG | 12275630 | ['12275630', '12275646', '12301585', '12306484', 1235044...| ['levothyroxine sodium 0.1 MG [levothyroxine sodium 0.1 MG]', 'levothyroxine sodium 0.01 MG [levothyroxine sodium 0.01 MG]', 'levo...| | 2 | 0 | aspirin 81 mg | DRUG | 12278696 | ['12278696', '12299811', '12298729', '12311168', '1230631...| ['aspirin 81 MG [aspirin 81 MG]', 'aspirin 81 MG [YSP Aspirin] [aspirin 81 MG [YSP Aspirin]]', 'aspirin 81 MG [Med Aspirin] [aspir...| | 3 | 0 | metformin 100 mg | DRUG | 12282749 | ['12282749', '3735316', '12279966', '1509573', '3736179'... | ['metformin hydrochloride 100 MG/ML [metformin hydrochloride 100 MG/ML]', 'metFORMIN hydrochloride 100 MG/ML [metFORMIN hydrochlor...| | 4 | 0 | coumadin 5 mg | DRUG | 1768579 | ['1768579', '12534260', '1780903', '1768951', '1510873' ... | ['coumarin 5 MG [coumarin 5 MG]', 'vericiguat 5 MG [vericiguat 5 MG]', 'pridinol 5 MG [pridinol 5 MG]', 'propinox 5 MG [propinox 5...| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_nih| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|818.8 MB| |Case sensitive:|false| ## References Trained on February 2023 with `sbiobert_base_cased_mli` embeddings. https://www.nlm.nih.gov/research/umls/rxnorm/docs/rxnormfiles.html --- layout: model title: Translate Gun to English Pipeline author: John Snow Labs name: translate_guw_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, guw, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `guw` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_guw_en_xx_2.7.0_2.4_1609688796603.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_guw_en_xx_2.7.0_2.4_1609688796603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_guw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_guw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.guw.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_guw_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Italian Named Entity Recognition (from gunghio) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_panx_ner date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, it, open_source] task: Named Entity Recognition language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-ner` is a Italian model orginally trained by `gunghio`. ## Predicted Entities `LOC`, `ORG`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_panx_ner_it_3.4.2_3.0_1652808069992.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_panx_ner_it_3.4.2_3.0_1652808069992.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_panx_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|it| |Size:|878.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/gunghio/xlm-roberta-base-finetuned-panx-ner --- layout: model title: Pipeline to Detect Normalized Genes and Human Phenotypes author: John Snow Labs name: ner_human_phenotype_gene_clinical_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, gene, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_gene_clinical](https://nlp.johnsnowlabs.com/2021/03/31/ner_human_phenotype_gene_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_clinical_pipeline_en_3.4.1_3.0_1647867667569.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_clinical_pipeline_en_3.4.1_3.0_1647867667569.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_human_phenotype_gene_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).") ``` ```scala val pipeline = new PretrainedPipeline("ner_human_phenotype_gene_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phnotype_gene_clinical.pipeline").predict("""Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).""") ```

## Results ```bash +----+------------------+---------+-------+----------+ | | chunk | begin | end | entity | +====+==================+=========+=======+==========+ | 0 | BS type | 29 | 32 | GENE | +----+------------------+---------+-------+----------+ | 1 | polyhydramnios | 75 | 88 | HP | +----+------------------+---------+-------+----------+ | 2 | polyuria | 91 | 98 | HP | +----+------------------+---------+-------+----------+ | 3 | nephrocalcinosis | 101 | 116 | HP | +----+------------------+---------+-------+----------+ | 4 | hypokalemia | 122 | 132 | HP | +----+------------------+---------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_gene_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739428334.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739428334.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_hier_quadruplet_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_hier_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: Chinese BertForMaskedLM Cased model (from qinluo) author: John Snow Labs name: bert_embeddings_wo_chinese_plus date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `wobert-chinese-plus` is a Chinese model originally trained by `qinluo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_wo_chinese_plus_zh_4.2.4_3.0_1670327320217.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_wo_chinese_plus_zh_4.2.4_3.0_1670327320217.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_wo_chinese_plus","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_wo_chinese_plus","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_wo_chinese_plus| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|467.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/qinluo/wobert-chinese-plus - https://github.com/ZhuiyiTechnology/WoBERT - https://github.com/JunnYu/WoBERT_pytorch --- layout: model title: Chunk Entity Resolver RxNorm-scdc author: John Snow Labs name: chunkresolve_rxnorm_scdc_healthcare date: 2021-04-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to RxNorm codes using chunk embeddings (augmented with synonyms, four times richer than previous resolver). ## Predicted Entities RxNorm codes {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_RXNORM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_scdc_healthcare_en_3.0.0_3.0_1618605170280.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_scdc_healthcare_en_3.0.0_3.0_1618605170280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... resolver = ChunkEntityResolverModel.pretrained("chunkresolve_rxnorm_scdc_healthcare","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") pipeline = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, resolver]) data = spark.createDataFrame([["A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation."]]).toDF("text") model = pipeline.fit(data) results = model.transform(data) ... ``` ```scala ... val resolver = ChunkEntityResolverModel.pretrained("chunkresolve_rxnorm_scdc_healthcare","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(document_assembler, sbert_embedder, resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation.").toDF("text") val result = pipeline.fit(data).transform(data) val result = pipeline.fit(data).transform(data) ```

## Results ```bash +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | chunk| entity| target_text| code|confidence| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | metformin|TREATMENT|metFORMIN compounding powder:::Metformin Hydrochloride Powder:::metFORMIN 500 mg oral tablet:::me...| 601021| 0.2364| | glipizide|TREATMENT|Glipizide Powder:::Glipizide Crystal:::Glipizide Tablets:::glipiZIDE 5 mg oral tablet:::glipiZIDE...| 241604| 0.3647| |dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG|TREATMENT|Ezetimibe and Atorvastatin Tablets:::Amlodipine and Atorvastatin Tablets:::Atorvastatin Calcium T...|1422084| 0.3407| | dapagliflozin|TREATMENT|Dapagliflozin Tablets:::dapagliflozin 5 mg oral tablet:::dapagliflozin 10 mg oral tablet:::Dapagl...|1488568| 0.7070| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_rxnorm_scdc_healthcare| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| --- layout: model title: Italian Electra Embeddings (from dbmdz) author: John Snow Labs name: electra_embeddings_electra_base_italian_xxl_cased_generator date: 2022-05-17 tags: [it, open_source, electra, embeddings] task: Embeddings language: it edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-italian-xxl-cased-generator` is a Italian model orginally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_italian_xxl_cased_generator_it_3.4.4_3.0_1652786574536.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_italian_xxl_cased_generator_it_3.4.4_3.0_1652786574536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_italian_xxl_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|it| |Size:|128.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Smaller BERT Embeddings (L-12_H-512_A-8) author: John Snow Labs name: small_bert_L12_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L12_512_en_2.6.0_2.4_1598344865471.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L12_512_en_2.6.0_2.4_1598344865471.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L12_512", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L12_512", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L12_512').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L12_512_embeddings I [0.5089142322540283, -0.21703988313674927, -0.... love [-0.3273950517177582, 0.9550480842590332, -0.1... NLP [0.3552919626235962, 0.3629235625267029, 0.891... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L12_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1 --- layout: model title: Arabic Bert Embeddings (Base, Arabert Model) author: John Snow Labs name: bert_embeddings_bert_base_arabert date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabert` is a Arabic model orginally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabert_ar_3.4.2_3.0_1649677303708.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabert_ar_3.4.2_3.0_1649677303708.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|507.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-base-arabert - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-base-arabert/resolve/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: Pipeline to Summarize Clinical Question Notes author: John Snow Labs name: summarizer_clinical_questions_pipeline date: 2023-05-31 tags: [licensed, en, clinical, summarization, question] task: Summarization language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_questions](https://nlp.johnsnowlabs.com/2023/04/03/summarizer_clinical_questions_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_pipeline_en_4.4.1_3.0_1685530642775.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_questions_pipeline_en_4.4.1_3.0_1685530642775.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_questions_pipeline", "en", "clinical/models") text = """ Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_questions_pipeline", "en", "clinical/models") val text = """ Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash What are the treatments for hyperthyroidism? ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_questions_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|936.7 MB| ## Included Models - DocumentAssembler - MedicalSummarizer --- layout: model title: Word2Vec Embeddings in Aragonese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, an, open_source] task: Embeddings language: an edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_an_3.4.1_3.0_1647282522053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_an_3.4.1_3.0_1647282522053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python news_pipeline = PretrainedPipeline("bert_sequence_classifier_age_news_pipeline", lang = "en") news_pipeline.annotate("Microsoft has taken its first step into the metaverse.") ``` ```scala val news_pipeline = new PretrainedPipeline("bert_sequence_classifier_age_news_pipeline", lang = "en") news_pipeline.annotate("Microsoft has taken its first step into the metaverse.") ```

## Results ```bash ['Sci/Tech'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_age_news_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|42.4 MB| ## Included Models - DocumentAssembler - TokenizerModel - BertForSequenceClassification --- layout: model title: English asr_wav2vec2_ksponspeech TFWav2Vec2ForCTC from Taeham author: John Snow Labs name: pipeline_asr_wav2vec2_ksponspeech date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_ksponspeech` is a English model originally trained by Taeham. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_ksponspeech_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_ksponspeech_en_4.2.0_3.0_1664102640740.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_ksponspeech_en_4.2.0_3.0_1664102640740.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_ksponspeech', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_ksponspeech", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_ksponspeech| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from efederici) author: John Snow Labs name: t5_it5_efficient_small_lfqa date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-lfqa` is a Italian model originally trained by `efederici`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_lfqa_it_4.3.0_3.0_1675103827826.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_lfqa_it_4.3.0_3.0_1675103827826.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_lfqa","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_lfqa","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_lfqa| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|594.0 MB| ## References - https://huggingface.co/efederici/it5-efficient-small-lfqa --- layout: model title: Legal NER (Parties, Dates, Document Type - sm) author: John Snow Labs name: legner_contract_doc_parties date: 2022-08-16 tags: [en, legal, ner, agreements, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_introduction_clause` Text Classifier to select only these paragraphs; This is a Legal NER Model, aimed to process the first page of the agreements when information can be found about: - Parties of the contract/agreement; - Aliases of those parties, or how those parties will be called further on in the document; - Document Type; - Effective Date of the agreement; This model can be used all along with its Relation Extraction model to retrieve the relations between these entities, called `legre_contract_doc_parties` Other models can be found to detect other parts of the document, as Headers/Subheaders, Signers, "Will-do", etc. ## Predicted Entities `PARTY`, `EFFDATE`, `DOC`, `ALIAS` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/LEGALNER_PARTIES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_contract_doc_parties_en_1.0.0_3.2_1660647946284.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_contract_doc_parties_en_1.0.0_3.2_1660647946284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------+---------+ | token|ner_label| +------------+---------+ |INTELLECTUAL| B-DOC| | PROPERTY| I-DOC| | AGREEMENT| I-DOC| | This| O| |INTELLECTUAL| B-DOC| | PROPERTY| I-DOC| | AGREEMENT| I-DOC| | (| O| | this| O| | "| O| | Agreement| O| | "),| O| | dated| O| | as| O| | of| O| | December|B-EFFDATE| | 31|I-EFFDATE| | ,|I-EFFDATE| | 2018|I-EFFDATE| | (| O| | the| O| | "| O| | Effective| O| | Date| O| | ")| O| | is| O| | entered| O| | into| O| | by| O| | and| O| | between| O| | Armstrong| B-PARTY| | Flooring| I-PARTY| | ,| I-PARTY| | Inc| I-PARTY| | .,| O| | a| O| | Delaware| O| | corporation| O| | ("| O| | Seller| B-ALIAS| | ")| O| | and| O| | AFI| B-PARTY| | Licensing| I-PARTY| | LLC| I-PARTY| | ,| O| | a| O| | Delaware| O| | limited| O| | liability| O| | company| O| | ("| O| | Licensing| B-ALIAS| | "| O| | and| O| | together| O| | with| O| | Seller| B-ALIAS| | ,| O| | "| O| | Arizona| B-ALIAS| | ")| O| | and| O| | AHF| B-PARTY| | Holding| I-PARTY| | ,| I-PARTY| | Inc| I-PARTY| | .| O| | (| O| | formerly| O| | known| O| | as| O| | Tarzan| O| | HoldCo| O| | ,| O| | Inc| O| | .),| O| | a| O| | Delaware| O| | corporation| O| | ("| O| | Buyer| B-ALIAS| | ")| O| | and| O| | Armstrong| B-PARTY| | Hardwood| I-PARTY| | Flooring| I-PARTY| | Company| I-PARTY| ------------------------ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_contract_doc_parties| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.5 MB| ## References Manual annotations on CUAD dataset ## Benchmarking ```bash label tp fp fn prec rec f1 I-PARTY 262 20 61 0.92907804 0.8111455 0.8661157 B-EFFDATE 22 4 9 0.84615386 0.7096774 0.77192986 B-DOC 38 4 12 0.9047619 0.76 0.82608694 I-EFFDATE 95 9 19 0.91346157 0.8333333 0.8715596 I-DOC 93 12 5 0.8857143 0.9489796 0.9162561 B-PARTY 88 10 29 0.8979592 0.75213677 0.81860465 B-ALIAS 64 7 14 0.90140843 0.82051283 0.8590604 ``` --- layout: model title: English RobertaForQuestionAnswering Tiny Cased model (from deepset) author: John Snow Labs name: roberta_qa_tiny_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tinyroberta-squad2` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_tiny_squad2_en_4.2.4_3.0_1669988560342.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_tiny_squad2_en_4.2.4_3.0_1669988560342.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_tiny_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_tiny_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_tiny_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|307.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/deepset/tinyroberta-squad2 - https://haystack.deepset.ai/tutorials/first-qa-system - https://arxiv.org/pdf/1909.10351.pdf - https://github.com/deepset-ai/haystack - https://github.com/deepset-ai/haystack/ - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - http://deepset.ai/ - https://haystack.deepset.ai/ - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/haystack - https://docs.haystack.deepset.ai - https://haystack.deepset.ai/community/join - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs - https://paperswithcode.com/sota?task=Question+Answering&dataset=squad_v2 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_768 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_768_zh_4.2.4_3.0_1670325955053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_768_zh_4.2.4_3.0_1670325955053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|170.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: English Biomedical ElectraForQuestionAnswering model author: John Snow Labs name: electra_qa_BioM_Base_SQuAD2_BioASQ8B date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true recommended: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Biomedical Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BioM-ELECTRA-Base-SQuAD2-BioASQ8B` is a English model originally trained by `sultan`. This model is fine-tuned on the SQuAD2.0 dataset and then on the BioASQ8B-Factoid training dataset. We convert the BioASQ8B-Factoid training dataset to SQuAD1.1 format and train and evaluate our model (BioM-ELECTRA-Base-SQuAD2) on this dataset. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Base_SQuAD2_BioASQ8B_en_4.0.0_3.0_1655918942331.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Base_SQuAD2_BioASQ8B_en_4.0.0_3.0_1655918942331.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_BioM_Base_SQuAD2_BioASQ8B","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_BioM_Base_SQuAD2_BioASQ8B","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_bioasq8b.electra.base").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_BioM_Base_SQuAD2_BioASQ8B| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sultan/BioM-ELECTRA-Base-SQuAD2-BioASQ8B --- layout: model title: Fast and Accurate Language Identification - 43 Languages (CNN) author: John Snow Labs name: ld_wiki_tatoeba_cnn_43 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Arabic`, `Belarusian`, `Bulgarian`, `Czech`, `Danish`, `German`, `Greek`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Persian`, `Finnish`, `French`, `Hebrew`, `Hindi`, `Hungarian`, `Interlingua`, `Indonesian`, `Icelandic`, `Italian`, `Japanese`, `Korean`, `Latin`, `Lithuanian`, `Latvian`, `Macedonian`, `Marathi`, `Dutch`, `Polish`, `Portuguese`, `Romanian`, `Russian`, `Slovak`, `Slovenian`, `Serbian`, `Swedish`, `Tagalog`, `Turkish`, `Tatar`, `Ukrainian`, `Vietnamese`, `Chinese`. ## Predicted Entities `ar`, `be`, `bg`, `cs`, `da`, `de`, `el`, `en`, `eo`, `es`, `et`, `fa`, `fi`, `fr`, `he`, `hi`, `hu`, `ia`, `id`, `is`, `it`, `ja`, `ko`, `la`, `lt`, `lv`, `mk`, `mr`, `nl`, `pl`, `pt`, `ro`, `ru`, `sk`, `sl`, `sr`, `sv`, `tl`, `tr`, `tt`, `uk`, `vi`, `zh`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_43_xx_2.7.0_2.4_1607184003726.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_43_xx_2.7.0_2.4_1607184003726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_43", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_43", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_43').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_tatoeba_cnn_43| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia and Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | fr| 1000| 1000| 1.0| | nl| 1000| 999| 0.999| | sv| 1000| 999| 0.999| | pt| 1000| 999| 0.999| | it| 1000| 999| 0.999| | es| 1000| 999| 0.999| | fi| 1000| 999| 0.999| | el| 1000| 998| 0.998| | de| 1000| 997| 0.997| | da| 1000| 997| 0.997| | en| 1000| 995| 0.995| | lt| 1000| 986| 0.986| | hu| 880| 867|0.9852272727272727| | pl| 914| 899|0.9835886214442013| | ro| 784| 765|0.9757653061224489| | et| 928| 899| 0.96875| | cs| 1000| 967| 0.967| | sk| 1000| 966| 0.966| | bg| 1000| 960| 0.96| | sl| 914| 860|0.9409190371991247| | lv| 916| 856|0.9344978165938864| +--------+-----+-------+------------------+ +-------+--------------------+ |summary| precision| +-------+--------------------+ | count| 21| | mean| 0.9832737168612825| | stddev|0.020064155103808722| | min| 0.9344978165938864| | max| 1.0| +-------+--------------------+ ``` --- layout: model title: Legal Law Area Prediction Classifier (French) author: John Snow Labs name: legclf_law_area_prediction_french date: 2023-03-29 tags: [fr, licensed, classification, legal, tensorflow] task: Text Classification language: fr edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model which identifies law area labels(civil_law, penal_law, public_law, social_law) in French-based Court Cases. ## Predicted Entities `civil_law`, `penal_law`, `public_law`, `social_law` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_law_area_prediction_french_fr_1.0.0_3.0_1680094841099.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_law_area_prediction_french_fr_1.0.0_3.0_1680094841099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_multi_cased", "xx")\ .setInputCols(["document"]) \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_law_area_prediction_french", "fr", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, docClassifier ]) df = spark.createDataFrame([["par ces motifs, le Juge unique prononce : 1. Le recours est irrecevable. 2. Il n'est pas perçu de frais judiciaires. 3. Le présent arrêt est communiqué aux parties, au Tribunal administratif fédéral et à l'Office fédéral des assurances sociales. Lucerne, le 2 juin 2016 Au nom de la IIe Cour de droit social du Tribunal fédéral suisse Le Juge unique : Meyer Le Greffier : Cretton"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) result.select("text", "category.result").show(truncate=100) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+------------+ | text| result| +----------------------------------------------------------------------------------------------------+------------+ |par ces motifs, le Juge unique prononce : 1. Le recours est irrecevable. 2. Il n'est pas perçu de...|[social_law]| +----------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_law_area_prediction_french| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|fr| |Size:|22.3 MB| ## References Train dataset available [here](https://huggingface.co/datasets/rcds/legal_criticality_prediction) ## Benchmarking ```bash label precision recall f1-score support civil_law 0.93 0.91 0.92 613 penal_law 0.94 0.96 0.95 579 public_law 0.92 0.91 0.92 605 social_law 0.97 0.98 0.97 478 accuracy - - 0.94 2275 macro-avg 0.94 0.94 0.94 2275 weighted-avg 0.94 0.94 0.94 2275 ``` --- layout: model title: English BertForQuestionAnswering model (from hendrixcosta) author: John Snow Labs name: bert_qa_bertimbau_squad1.1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertimbau-squad1.1` is a English model orginally trained by `hendrixcosta`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bertimbau_squad1.1_en_4.0.0_3.0_1654185392313.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bertimbau_squad1.1_en_4.0.0_3.0_1654185392313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bertimbau_squad1.1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bertimbau_squad1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_hendrixcosta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bertimbau_squad1.1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/hendrixcosta/bertimbau-squad1.1 --- layout: model title: Classification of Self-Reported Intimate Partner Violence (BioBERT) author: John Snow Labs name: bert_sequence_classifier_self_reported_partner_violence_tweet date: 2022-07-28 tags: [sequence_classification, bert, classifier, clinical, en, licensed, public_health, partner_violence, tweet] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classification of Self-Reported Intimate Partner Violence on Twitter. This model involves the detection the potential IPV victims on social media platforms (in English tweets). ## Predicted Entities `intimate_partner_violence`, `non-intimate_partner_violence` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_PARTNER_VIOLENCE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_self_reported_partner_violence_tweet_en_4.0.0_3.0_1658999356448.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_self_reported_partner_violence_tweet_en_4.0.0_3.0_1658999356448.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_self_reported_partner_violence_tweet", "en", "clinical/models")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) example = spark.createDataFrame(["I am fed up with this toxic relation.I hate my husband.", "Can i say something real quick I ve never been one to publicly drag an ex partner and sometimes I regret that. I ve been reflecting on the harm, abuse and violence that was done to me and those bitches are truly lucky I chose peace amp therapy because they are trash forreal."], StringType()).toDF("text") result = pipeline.fit(example).transform(example) result.select("text", "class.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_self_reported_partner_violence_tweet", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) # couple of simple examples val example = Seq(Array("I am fed up with this toxic relation.I hate my husband.", "Can i say something real quick I ve never been one to publicly drag an ex partner and sometimes I regret that. I ve been reflecting on the harm, abuse and violence that was done to me and those bitches are truly lucky I chose peace amp therapy because they are trash forreal.")).toDF("text") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.self_reported_partner_violence").predict("""Can i say something real quick I ve never been one to publicly drag an ex partner and sometimes I regret that. I ve been reflecting on the harm, abuse and violence that was done to me and those bitches are truly lucky I chose peace amp therapy because they are trash forreal.""") ```

## Results ```bash +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------+ |text |result | +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------+ |I am fed up with this toxic relation.I hate my husband. |[non-intimate_partner_violence]| |Can i say something real quick I ve never been one to publicly drag an ex partner and sometimes I regret that. I ve been reflecting on the harm, abuse and violence that was done to me and those bitches are truly lucky I chose peace amp therapy because they are trash forreal.|[intimate_partner_violence] | +-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_self_reported_partner_violence_tweet| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References [SMM4H 2022](https://healthlanguageprocessing.org/smm4h-2022/) ## Benchmarking ```bash label precision recall f1-score support intimate_partner_violence 0.96 0.97 0.97 630 non-intimate_partner_violence 0.75 0.69 0.72 78 accuracy - - 0.94 708 macro-avg 0.86 0.83 0.84 708 weighted-avg 0.94 0.94 0.94 708 ``` --- layout: model title: Clinical Deidentification (Spanish) author: John Snow Labs name: clinical_deidentification date: 2022-02-17 tags: [deid, es, licensed,clinical] task: De-identification language: es edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with sciwiki_300d embeddings and can be used to deidentify PHI information from medical texts in Spanish. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask, fake or obfuscate the following entities: `AGE`, `DATE`, `PROFESSION`, `E-MAIL`, `USERNAME`, `LOCATION`, `DOCTOR`, `HOSPITAL`, `PATIENT`, `URL`, `IP`, `MEDICALRECORD`, `IDNUM`, `ORGANIZATION`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `PLATE`, `SEX` and `IPADDR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.4.1_3.0_1645118722536.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.4.1_3.0_1645118722536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Masked with entity labels ------------------------------ Datos del paciente. Nombre: . Apellidos: . NHC: . NASS: 04. Domicilio: , 5 B.. Localidad/ Provincia: . CP: . Datos asistenciales. Fecha de nacimiento: . País: . Edad: años Sexo: . Fecha de Ingreso: . : María Merino Viveros NºCol: . Informe clínico del paciente: de años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Servicio de Endocrinología y Nutrición km 12,500 28905 - () Correo electrónico: Masked with chars ------------------------------ Datos del paciente. Nombre: [**] . Apellidos: [*************]. NHC: [*****]. NASS: ** [******] 04. Domicilio: [*******************], 5 B.. Localidad/ Provincia: [****]. CP: [***]. Datos asistenciales. Fecha de nacimiento: [********]. País: [****]. Edad: ** años Sexo: *. Fecha de Ingreso: [********]. [****]: María Merino Viveros NºCol: ** ** [***]. Informe clínico del paciente: [***] de ** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en [*********] en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: [*************] +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente [****] significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. [******************] [******************************] Servicio de Endocrinología y Nutrición [*****************] km 12,500 28905 [****] - [****] ([****]) Correo electrónico: [********************] Masked with fixed length chars ------------------------------ Datos del paciente. Nombre: **** . Apellidos: ****. NHC: ****. NASS: **** **** 04. Domicilio: ****, 5 B.. Localidad/ Provincia: ****. CP: ****. Datos asistenciales. Fecha de nacimiento: ****. País: ****. Edad: **** años Sexo: ****. Fecha de Ingreso: ****. ****: María Merino Viveros NºCol: **** **** ****. Informe clínico del paciente: **** de **** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en **** en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: **** +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente **** significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. **** **** Servicio de Endocrinología y Nutrición **** km 12,500 28905 **** - **** (****) Correo electrónico: **** Obfuscated ------------------------------ Datos del paciente. Nombre: Sr. Lerma . Apellidos: Aristides Gonzalez Gelabert. NHC: BBBBBBBBQR648597. NASS: 041010000011 RZRM020101906017 04. Domicilio: Valencia, 5 B.. Localidad/ Provincia: Madrid. CP: 99335. Datos asistenciales. Fecha de nacimiento: 25/04/1977. País: Barcelona. Edad: 8 años Sexo: F.. Fecha de Ingreso: 02/08/2018. transportista: María Merino Viveros NºCol: olegario10 olegario10 felisa78. Informe clínico del paciente: RZRM020101906017 de 8 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Madrid en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Dra. Laguna +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente 041010000011 significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Reinaldo Manjón Malo Barcelona Servicio de Endocrinología y Nutrición Valencia km 12,500 28905 Bilbao - Madrid (Barcelona) Correo electrónico: quintanasalome@example.net ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|281.3 MB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetectorDLModel - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - nlp.NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: English BertForQuestionAnswering model (from gerardozq) author: John Snow Labs name: bert_qa_biobert_v1.1_pubmed_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert_v1.1_pubmed-finetuned-squad` is a English model orginally trained by `gerardozq`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_v1.1_pubmed_finetuned_squad_en_4.0.0_3.0_1654185735686.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_v1.1_pubmed_finetuned_squad_en_4.0.0_3.0_1654185735686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_v1.1_pubmed_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_v1.1_pubmed_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_pubmed.biobert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_v1.1_pubmed_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/gerardozq/biobert_v1.1_pubmed-finetuned-squad --- layout: model title: Hocr for table recognition author: John Snow Labs name: hocr_table_recognition date: 2023-01-23 tags: [en, licensed] task: HOCR Table Recognition language: en nav_key: models edition: Visual NLP 4.2.4 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Table structure recognition based on hocr with Tesseract architecture. Tesseract has been trained on a variety of datasets to improve its recognition capabilities. These datasets include images of text in various languages and scripts, as well as images with different font styles, sizes, and orientations. The training process involves feeding the engine with a large number of images and their corresponding text, allowing the engine to learn the patterns and characteristics of different text styles. One of the most important datasets used in training Tesseract is the UNLV dataset, which contains over 400,000 images of text in different languages, scripts, and font styles. This dataset is widely used in the OCR community and has been instrumental in improving the accuracy of Tesseract. Other datasets that have been used in training Tesseract include the ICDAR dataset, the IIIT-HWS dataset, and the RRC-GV-WS dataset. In addition to these datasets, Tesseract also uses a technique called adaptive training, where the engine can continuously improve its recognition capabilities by learning from new images and text. This allows Tesseract to adapt to new text styles and languages, and improve its overall accuracy. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/IMAGE_TABLE_RECOGNITION_HOCR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/tree/master/jupyter/SparkOcrImageTableRecognitionWHOCR.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") table_detector = ImageTableDetector.pretrained("general_model_table_detection_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("table_regions") splitter = ImageSplitRegions() \ .setInputCol("image") \ .setInputRegionsCol("table_regions") \ .setOutputCol("table_image") \ .setDropCols("image") \ .setImageType(ImageType.TYPE_BYTE_GRAY) \ .setExplodeCols([]) text_detector = ImageTextDetectorV2.pretrained("image_text_detector_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("text_regions") \ .setWithRefiner(True) draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("text_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.green) \ .setRotated(True) img_to_hocr = ImageToTextV2().pretrained("ocr_small_printed", "en", "clinical/ocr") \ .setInputCols(["image", "text_regions"]) \ .setUsePandasUdf(False) \ .setOutputFormat(OcrOutputFormat.HOCR) \ .setOutputCol("hocr") \ .setGroupImages(False) hocr_to_table = HocrToTextTable() \ .setInputCol("hocr") \ .setRegionCol("table_regions") \ .setOutputCol("tables") pipeline = PipelineModel(stages=[ binary_to_image, table_detector, splitter, text_detector, draw_regions, img_to_hocr, hocr_to_table ]) imagePath = "data/tab_images_hocr_1/table4_1.jpg" image_df= spark.read.format("binaryFile").load(imagePath) result = pipeline.transform(image_df).cache() ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") val table_detector = new ImageTableDetector .pretrained("general_model_table_detection_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("table_regions") val splitter = new ImageSplitRegions() .setInputCol("image") .setInputRegionsCol("table_regions") .setOutputCol("table_image") .setDropCols("image") .setImageType(ImageType.TYPE_BYTE_GRAY) .setExplodeCols(Array()) val text_detector = new ImageTextDetectorV2 .pretrained("image_text_detector_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("text_regions") .setWithRefiner(True) val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("text_regions") .setOutputCol("image_with_regions") .setRectColor(Color.green) .setRotated(True) img_to_hocr = ImageToTextV2() .pretrained("ocr_small_printed", "en", "clinical/ocr") .setInputCols(Array("image", "text_regions")) .setUsePandasUdf(False) .setOutputFormat(OcrOutputFormat.HOCR) .setOutputCol("hocr") .setGroupImages(False) val hocr_to_table = new HocrToTextTable() .setInputCol("hocr") .setRegionCol("table_regions") .setOutputCol("tables") val pipeline = new PipelineModel().setStages(Array( binary_to_image, table_detector, splitter, text_detector, draw_regions, img_to_hocr, hocr_to_table)) val imagePath = "data/tab_images_hocr_1/table4_1.jpg" val image_df= spark.read.format("binaryFile").load(imagePath) val result = pipeline.transform(image_df).cache() ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image13.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image13_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash text_regions table_image pagenum modificationTime path table_regions length image image_with_regions hocr tables exception table_index [{0, 0, 566.32025... {file:/content/ta... 0 2023-01-23 08:21:... file:/content/tab... {0, 0, 40.0, 0.0,... 172124 {file:/content/ta... {file:/content/ta... Live Demo [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_12_en_4.1.0_3.0_1660171578567.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_12_en_4.1.0_3.0_1660171578567.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_12", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_12", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_12| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Mongolian RobertaForTokenClassification Base Cased model (from onon214) author: John Snow Labs name: roberta_token_classifier_base_ner_demo date: 2023-03-01 tags: [mn, open_source, roberta, token_classification, ner, tensorflow] task: Named Entity Recognition language: mn edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-ner-demo` is a Mongolian model originally trained by `onon214`. ## Predicted Entities `MISC`, `LOC`, `PER`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_base_ner_demo_mn_4.3.0_3.0_1677703536380.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_base_ner_demo_mn_4.3.0_3.0_1677703536380.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_base_ner_demo","mn") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_base_ner_demo","mn") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_base_ner_demo| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|mn| |Size:|466.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/onon214/roberta-base-ner-demo --- layout: model title: Fast Neural Machine Translation Model from Igbo to English author: John Snow Labs name: opus_mt_ig_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ig, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ig` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ig_en_xx_2.7.0_2.4_1609163903644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ig_en_xx_2.7.0_2.4_1609163903644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ig_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Right of setoff Clause Binary Classifier author: John Snow Labs name: legclf_right_of_setoff_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `right-of-setoff` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `right-of-setoff` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_right_of_setoff_clause_en_1.0.0_3.2_1660122968803.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_right_of_setoff_clause_en_1.0.0_3.2_1660122968803.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_right_of_setoff_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[right-of-setoff]| |[other]| |[other]| |[right-of-setoff]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_right_of_setoff_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.99 0.97 94 right-of-setoff 0.94 0.77 0.85 22 accuracy - - 0.95 116 macro-avg 0.95 0.88 0.91 116 weighted-avg 0.95 0.95 0.95 116 ``` --- layout: model title: Legal Erisa Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_erisa_bert date: 2023-03-05 tags: [en, legal, classification, clauses, erisa, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Erisa` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Erisa`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_erisa_bert_en_1.0.0_3.0_1678050577180.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_erisa_bert_en_1.0.0_3.0_1678050577180.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_erisa_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Erisa]| |[Other]| |[Other]| |[Erisa]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_erisa_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Erisa 0.97 1.00 0.99 35 Other 1.00 0.98 0.99 56 accuracy - - 0.99 91 macro-avg 0.99 0.99 0.99 91 weighted-avg 0.99 0.99 0.99 91 ``` --- layout: model title: Turkish ElectraForQuestionAnswering model (from enelpi) Discriminator Version-2 author: John Snow Labs name: electra_qa_base_discriminator_finetuned_squadv2 date: 2022-06-22 tags: [tr, open_source, electra, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-discriminator-finetuned_squadv2_tr` is a Turkish model originally trained by `enelpi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv2_tr_4.0.0_3.0_1655920605376.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv2_tr_4.0.0_3.0_1655920605376.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv2","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv2","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.squadv2.electra.base_v2").predict("""Benim adım ne?|||"Benim adım Clara ve Berkeley'de yaşıyorum.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_discriminator_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|412.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/enelpi/electra-base-discriminator-finetuned_squadv2_tr --- layout: model title: English RoBERTa Embeddings (from abhi1nandy2) author: John Snow Labs name: roberta_embeddings_Bible_roberta_base date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `Bible-roberta-base` is a English model orginally trained by `abhi1nandy2`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_Bible_roberta_base_en_3.4.2_3.0_1649947380949.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_Bible_roberta_base_en_3.4.2_3.0_1649947380949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_Bible_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|468.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/abhi1nandy2/Bible-roberta-base - https://www.kaggle.com/oswinrh/bible --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_ft_news date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta_FT_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ft_news_en_4.3.0_3.0_1674222981909.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ft_news_en_4.3.0_3.0_1674222981909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ft_news","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ft_news","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ft_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|458.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/roberta_FT_newsqa --- layout: model title: English asr_wav2vec2_cetuc_sid_voxforge_mls_0 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_cetuc_sid_voxforge_mls_0` is a English model originally trained by joaoalvarenga. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0_en_4.2.0_3.0_1664022807196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0_en_4.2.0_3.0_1664022807196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_0| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Part of Speech for Chinese author: John Snow Labs name: pos_ctb9 date: 2021-01-03 task: Part of Speech Tagging language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [pos, zh, cn, open_source] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PN (pronoun), CC (coordinating conjunction), and 39 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. ## Predicted Entities `AD`, `AS`, `BA`, `CC`, `CD`, `CS`, `DEC`, `DEG`, `DER`, `DEV`, `DT`, `EM`, `ETC`, `FW`, `IC`, `IJ`, `JJ`, `LB`, `LC`, `M`, `MSP`, `MSP-2`, `NN`, `NN-SHORT`, `NOI`, `NR`, `NR-SHORT`, `NT`, `NT-SHORT`, `OD`, `ON`, `P`, `PN`, `PU`, `SB`, `SP`, `URL`, `VA`, `VC`, `VE`, and `VV` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ctb9_zh_2.7.0_2.4_1609696404134.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ctb9_zh_2.7.0_2.4_1609696404134.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_large", "zh")\ .setInputCols(["sentence"])\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_ctb9", zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, pos]) example = spark.createDataFrame([['然而，这样的处理也衍生了一些问题。']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_large", "zh") .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_ctb9", "zh") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, word_segmenter, pos)) val data = Seq("然而，这样的处理也衍生了一些问题。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，这样的处理也衍生了一些问题。"""] pos_df = nlu.load('zh.pos.ctb9').predict(text, output_level='token') pos_df ```

## Results ```bash +-----+---+ |token|pos_tag| +-----+---+ |然而 |AD | |, |PU | |这样 |PN | |的 |DEG| |处理 |NN | |也 |AD | |衍生 |VV | |了 |AS | |一些 |CD | |问题 |NN | |。 |PU | +-----+---+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ctb9| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|zh| ## Data Source The model was trained on the Microsoft Research Asia (MSRA) data set available on the Second International Chinese Word Segmentation Bakeoff [SIGHAN 2005](http://sighan.cs.uchicago.edu/bakeoff2005/) ## Benchmarking ```bash | Tag | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | AD | 0.92 | 0.94 | 0.93 | 23017 | | AS | 0.92 | 0.93 | 0.92 | 2415 | | BA | 0.95 | 0.94 | 0.95 | 376 | | CC | 0.85 | 0.82 | 0.83 | 2388 | | CD | 0.95 | 0.95 | 0.95 | 6758 | | CS | 0.93 | 0.92 | 0.92 | 525 | | DEC | 0.75 | 0.78 | 0.76 | 4422 | | DEG | 0.86 | 0.85 | 0.85 | 6094 | | DER | 0.87 | 0.79 | 0.83 | 251 | | DEV | 0.86 | 0.73 | 0.79 | 291 | | DT | 0.94 | 0.91 | 0.92 | 4409 | | EM | 0.74 | 0.72 | 0.73 | 32 | | ETC | 0.97 | 0.98 | 0.97 | 294 | | FW | 0.00 | 0.00 | 0.00 | 2 | | IC | 0.11 | 0.01 | 0.02 | 117 | | IJ | 0.95 | 0.95 | 0.95 | 2153 | | JJ | 0.80 | 0.77 | 0.78 | 4693 | | LB | 0.78 | 0.70 | 0.73 | 105 | | LC | 0.95 | 0.97 | 0.96 | 2660 | | M | 0.97 | 0.97 | 0.97 | 6467 | | MSP | 0.83 | 0.82 | 0.83 | 428 | | MSP-2 | 0.00 | 0.00 | 0.00 | 1 | | NN | 0.93 | 0.94 | 0.93 | 49159 | | NN-SHORT | 0.00 | 0.00 | 0.00 | 1 | | NOI | 0.17 | 0.06 | 0.09 | 48 | | NR | 0.95 | 0.90 | 0.93 | 13220 | | NR-SHORT | 0.00 | 0.00 | 0.00 | 8 | | NT | 0.95 | 0.95 | 0.95 | 3723 | | NT-SHORT | 1.00 | 0.20 | 0.33 | 5 | | OD | 0.91 | 0.86 | 0.88 | 587 | | ON | 0.33 | 0.23 | 0.27 | 13 | | P | 0.90 | 0.92 | 0.91 | 7442 | | PN | 0.95 | 0.95 | 0.95 | 11011 | | PU | 0.99 | 0.99 | 0.99 | 36717 | | SB | 0.89 | 0.90 | 0.90 | 197 | | SP | 0.91 | 0.90 | 0.91 | 5466 | | URL | 0.75 | 0.75 | 0.75 | 8 | | VA | 0.84 | 0.83 | 0.83 | 4866 | | VC | 0.98 | 0.98 | 0.98 | 4434 | | VE | 0.95 | 0.95 | 0.95 | 2274 | | VV | 0.91 | 0.91 | 0.91 | 35240 | | | | | | | | accuracy | 0.93 | 242317 | | | | macro avg | 0.76 | 0.72 | 0.73 | 242317 | | weighted avg | 0.93 | 0.93 | 0.93 | 242317 | ``` --- layout: model title: Pipeline to Detect Anatomical and Observation Entities in Chest Radiology Reports author: John Snow Labs name: ner_chexpert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, chexpert, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chexpert](https://nlp.johnsnowlabs.com/2021/09/30/ner_chexpert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_pipeline_en_3.4.1_3.0_1647867766035.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chexpert_pipeline_en_3.4.1_3.0_1647867766035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_chexpert_pipeline", "en", "clinical/models") pipeline.annotate("FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax . FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.") ``` ```scala val pipeline = new PretrainedPipeline("ner_chexpert_pipeline", "en", "clinical/models") pipeline.annotate("FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax . FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chexpert.pipeline").predict("""FINAL REPORT HISTORY : Chest tube leak , to assess for pneumothorax . FINDINGS : In comparison with study of ___ , the endotracheal tube and Swan - Ganz catheter have been removed . The left chest tube remains in place and there is no evidence of pneumothorax. Mild atelectatic changes are seen at the left base.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bsc_temu_large_bne_s_c","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bsc_temu_large_bne_s_c","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_bsc_temu_large_bne_s_c| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/BSC-TeMU/roberta-large-bne-sqac - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://github.com/PlanTL-SANIDAD/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Bangla BertForQuestionAnswering model (from sagorsarker) author: John Snow Labs name: bert_qa_mbert_bengali_tydiqa_qa date: 2022-06-02 tags: [bn, open_source, question_answering, bert] task: Question Answering language: bn edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mbert-bengali-tydiqa-qa` is a Bangla model orginally trained by `sagorsarker`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_bengali_tydiqa_qa_bn_4.0.0_3.0_1654188244734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_bengali_tydiqa_qa_bn_4.0.0_3.0_1654188244734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_bengali_tydiqa_qa","bn") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_mbert_bengali_tydiqa_qa","bn") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("bn.answer_question.tydiqa.multi_lingual_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_bengali_tydiqa_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|bn| |Size:|626.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sagorsarker/mbert-bengali-tydiqa-qa - https://github.com/sagorbrur - https://github.com/sagorbrur/bntransformer - https://github.com/google-research-datasets/tydiqa - https://www.linkedin.com/in/sagor-sarker/ - https://www.kaggle.com/ --- layout: model title: Malay (macrolanguage) BertForQuestionAnswering model (from zhufy) author: John Snow Labs name: bert_qa_squad_ms_bert_base date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: ms edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-ms-bert-base` is a Malay (macrolanguage) model orginally trained by `zhufy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad_ms_bert_base_ms_4.0.0_3.0_1654192110158.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad_ms_bert_base_ms_4.0.0_3.0_1654192110158.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad_ms_bert_base","ms") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad_ms_bert_base","ms") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ms.answer_question.squad.bert.ms_tuned.base.by_zhufy").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad_ms_bert_base| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ms| |Size:|412.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/zhufy/squad-ms-bert-base - https://github.com/huseinzol05/malay-dataset/tree/master/question-answer/squad --- layout: model title: Russian Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_base_russian_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, ru, open_source] task: Part of Speech Tagging language: ru edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-russian-upos` is a Russian model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_russian_upos_ru_3.4.2_3.0_1652091813748.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_russian_upos_ru_3.4.2_3.0_1652091813748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_russian_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ru| |Size:|665.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-base-russian-upos - https://universaldependencies.org/ru/ - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Word Embeddings for Hindi (hindi_cc_300d) author: John Snow Labs name: hindi_cc_300d date: 2021-02-03 task: Embeddings language: hi edition: Spark NLP 2.7.2 spark_version: 2.4 tags: [embeddings, open_source, hi] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/hindi_cc_300d_hi_2.7.2_2.4_1612362695785.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/hindi_cc_300d_hi_2.7.2_2.4_1612362695785.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("hindi_cc_300d", "hi") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("hi.embed").predict("""Put your text here.""") ```

## Results ```bash The model gives 300 dimensional feature vector output per token. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|hindi_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.2+| |License:|Open Source| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|hi| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: English asr_wav2vec2_thai_ASR TFWav2Vec2ForCTC from Rattana author: John Snow Labs name: pipeline_asr_wav2vec2_thai_ASR date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_thai_ASR` is a English model originally trained by Rattana. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_thai_ASR_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_thai_ASR_en_4.2.0_3.0_1664112707199.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_thai_ASR_en_4.2.0_3.0_1664112707199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_thai_ASR', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_thai_ASR", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_thai_ASR| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten` is a English model originally trained by patrickvonplaten. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten_en_4.2.0_3.0_1664114081296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten_en_4.2.0_3.0_1664114081296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_google_colab_by_patrickvonplaten| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.3 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_temp TFWav2Vec2ForCTC from ying-tina author: John Snow Labs name: pipeline_asr_temp date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_temp` is a English model originally trained by ying-tina. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_temp_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_temp_en_4.2.0_3.0_1664110837777.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_temp_en_4.2.0_3.0_1664110837777.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_temp', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_temp", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_temp| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English ElectraForQuestionAnswering model (from sultan) author: John Snow Labs name: electra_qa_BioM_Base_SQuAD2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BioM-ELECTRA-Base-SQuAD2` is a English model originally trained by `sultan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Base_SQuAD2_en_4.0.0_3.0_1655918898262.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Base_SQuAD2_en_4.0.0_3.0_1655918898262.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_BioM_Base_SQuAD2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_BioM_Base_SQuAD2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.electra.base.by_sultan").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_BioM_Base_SQuAD2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sultan/BioM-ELECTRA-Base-SQuAD2 - https://github.com/salrowili/BioM-Transformers --- layout: model title: NER Pipeline for Clinical Problems (reduced taxonomy) - Voice of the Patient author: John Snow Labs name: ner_vop_problem_reduced_pipeline date: 2023-06-10 tags: [licensed, pipeline, ner, en, vop, problem] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline extracts mentions of clinical problems from health-related text in colloquial language. All problem entities are merged into one generic Problem class. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_reduced_pipeline_en_4.4.3_3.0_1686420051472.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_reduced_pipeline_en_4.4.3_3.0_1686420051472.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_vop_problem_reduced_pipeline", "en", "clinical/models") pipeline.annotate(" I've been experiencing joint pain and fatigue lately, so I went to the rheumatology department. After some tests, they diagnosed me with rheumatoid arthritis and started me on a treatment plan to manage the symptoms. ") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_vop_problem_reduced_pipeline", "en", "clinical/models") val result = pipeline.annotate(" I've been experiencing joint pain and fatigue lately, so I went to the rheumatology department. After some tests, they diagnosed me with rheumatoid arthritis and started me on a treatment plan to manage the symptoms. ") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |الرياض |LOC | |فيصل بن بندر بن عبد العزيز |PER | |الرياض |LOC | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_large_token_classifier_hrl| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|1.8 GB| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/Davlan/xlm-roberta-large-ner-hrl](https://huggingface.co/Davlan/xlm-roberta-large-ner-hrl) --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Original_SciBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Original-SciBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_SciBERT_384_en_4.0.0_3.0_1657108742652.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_SciBERT_384_en_4.0.0_3.0_1657108742652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Original_SciBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Original-SciBERT-384 --- layout: model title: Legal Agreement and Plan of Reorganization Document Classifier (Longformer) author: John Snow Labs name: legclf_agreement_and_plan_of_reorganization date: 2022-12-06 tags: [en, legal, classification, agreement, reorganization, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_agreement_and_plan_of_reorganization` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `agreement-and-plan-of-reorganization` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `agreement-and-plan-of-reorganization`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agreement_and_plan_of_reorganization_en_1.0.0_3.0_1670357495950.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agreement_and_plan_of_reorganization_en_1.0.0_3.0_1670357495950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[agreement-and-plan-of-reorganization]| |[other]| |[other]| |[agreement-and-plan-of-reorganization]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agreement_and_plan_of_reorganization| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support agreement-and-plan-of-reorganization 1.00 0.92 0.96 52 other 0.97 1.00 0.98 111 accuracy - - 0.98 163 macro-avg 0.98 0.96 0.97 163 weighted-avg 0.98 0.98 0.98 163 ``` --- layout: model title: English Electra Embeddings (from google) author: John Snow Labs name: electra_embeddings_electra_large_generator date: 2022-05-17 tags: [en, open_source, electra, embeddings] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-large-generator` is a English model orginally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_large_generator_en_3.4.4_3.0_1652786652489.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_large_generator_en_3.4.4_3.0_1652786652489.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_large_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|192.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/google/electra-large-generator - https://arxiv.org/pdf/1406.2661.pdf - https://rajpurkar.github.io/SQuAD-explorer/ - https://openreview.net/pdf?id=r1xMH1BtvB - https://gluebenchmark.com/ - https://rajpurkar.github.io/SQuAD-explorer/ - https://www.clips.uantwerpen.be/conll2000/chunking/ --- layout: model title: English DistilBertForQuestionAnswering model (from aszidon) Custom Version-4 author: John Snow Labs name: distilbert_qa_custom4 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom4` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom4_en_4.0.0_3.0_1654728016252.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom4_en_4.0.0_3.0_1654728016252.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.custom4.by_aszidon").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom4 --- layout: model title: Entity Recognizer LG author: John Snow Labs name: entity_recognizer_lg date: 2022-06-25 tags: [ru, open_source] task: Named Entity Recognition language: ru edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_ru_4.0.0_3.0_1656125353536.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_ru_4.0.0_3.0_1656125353536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("entity_recognizer_lg", "ru") result = pipeline.annotate("""I love johnsnowlabs! """) ``` {:.nlu-block} ```python import nlu nlu.load("ru.ner.lg").predict("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|ru| |Size:|2.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: Legal Competition Document Classifier (EURLEX) author: John Snow Labs name: legclf_competition_bert date: 2023-03-06 tags: [en, legal, classification, clauses, competition, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_competition_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Competition or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Competition`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_competition_bert_en_1.0.0_3.0_1678111605884.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_competition_bert_en_1.0.0_3.0_1678111605884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_competition_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Competition]| |[Other]| |[Other]| |[Competition]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_competition_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Competition 0.93 0.87 0.9 363 Other 0.87 0.92 0.9 333 accuracy - - 0.9 696 macro-avg 0.90 0.90 0.9 696 weighted-avg 0.90 0.90 0.9 696 ``` --- layout: model title: Pipeline to Detect Adverse Drug Events (MedicalBertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_ade_binary_pipeline date: 2023-03-20 tags: [clinical, ade, licensed, public_health, token_classification, ner, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_ade_binary](https://nlp.johnsnowlabs.com/2022/07/27/bert_token_classifier_ner_ade_binary_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_binary_pipeline_en_4.3.0_3.2_1679299868936.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_binary_pipeline_en_4.3.0_3.2_1679299868936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_ade_binary_pipeline", "en", "clinical/models") text = '''I used to be on paxil but that made me more depressed and prozac made me angry, Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_ade_binary_pipeline", "en", "clinical/models") val text = "I used to be on paxil but that made me more depressed and prozac made me angry, Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------|--------:|------:|:------------|-------------:| | 0 | depressed | 44 | 52 | ADE | 0.990846 | | 1 | angry | 73 | 77 | ADE | 0.972025 | | 2 | sugar crashes | 147 | 159 | ADE | 0.933623 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_ade_binary_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Portuguese asr_bp_tedx100_xlsr TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_bp_tedx100_xlsr date: 2022-09-26 tags: [wav2vec2, pt, audio, open_source, asr] task: Automatic Speech Recognition language: pt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bp_tedx100_xlsr` is a Portuguese model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bp_tedx100_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bp_tedx100_xlsr_pt_4.2.0_3.0_1664192496842.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bp_tedx100_xlsr_pt_4.2.0_3.0_1664192496842.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bp_tedx100_xlsr", "pt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bp_tedx100_xlsr", "pt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bp_tedx100_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|pt| |Size:|756.0 MB| --- layout: model title: Detect Pathogen, Medical Condition and Medicine author: John Snow Labs name: ner_pathogen date: 2022-06-28 tags: [licensed, clinical, en, ner, pathogen, medical_condition, medicine] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained named entity recognition (NER) model is a deep learning model for detecting medical conditions (influenza, headache, malaria, etc), medicine (aspirin, penicillin, methotrexate) and pathogens (Corona Virus, Zika Virus, E. Coli, etc) in clinical texts. It is trained by using `MedicalNerApproach` annotator that allows to train generic NER models based on Neural Networks. ## Predicted Entities `Pathogen`, `MedicalCondition`, `Medicine` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_pathogen_en_4.0.0_3.0_1656419618392.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_pathogen_en_4.0.0_3.0_1656419618392.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------+----------------+ |chunk |ner_label | +---------------+----------------+ |Racecadotril |Medicine | |loperamide |Medicine | |Diarrhea |MedicalCondition| |dehydration |MedicalCondition| |skin color |MedicalCondition| |fast heart rate|MedicalCondition| |rabies virus |Pathogen | |Lyssavirus |Pathogen | |Ephemerovirus |Pathogen | +---------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_pathogen| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|14.6 MB| ## References Trained on [dataset](https://www.kaggle.com/datasets/finalepoch/medical-ner) to get a model for Named Entity Recognition. ## Benchmarking ```bash label tp fp fn total precision recall f1 Pathogen 15.0 3.0 9.0 24.0 0.8333 0.625 0.7143 Medicine 15.0 2.0 0.0 15.0 0.8824 1.0 0.9375 MedicalCondition 53.0 2.0 6.0 59.0 0.9636 0.8983 0.9298 macro - - - - - - 0.8605 micro - - - - - - 0.8782 ``` --- layout: model title: Russian BertForQuestionAnswering Cased model (from ruselkomp) author: John Snow Labs name: bert_qa_deep_pavlov_full date: 2022-07-07 tags: [ru, open_source, bert, question_answering] task: Question Answering language: ru edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deep-pavlov-full` is a Russian model originally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_deep_pavlov_full_ru_4.0.0_3.0_1657189256159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_deep_pavlov_full_ru_4.0.0_3.0_1657189256159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_deep_pavlov_full","ru") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Как меня зовут?", "Меня зовут Клара, и я живу в Беркли."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_deep_pavlov_full","ru") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Как меня зовут?", "Меня зовут Клара, и я живу в Беркли.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_deep_pavlov_full| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ru| |Size:|665.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/deep-pavlov-full --- layout: model title: Pipeline to Detect Living Species author: John Snow Labs name: ner_living_species_biobert_pipeline date: 2023-03-20 tags: [ner, en, clinical, licensed, biobert] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species_biobert](https://nlp.johnsnowlabs.com/2022/06/22/ner_living_species_biobert_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_biobert_pipeline_en_4.3.0_3.2_1679309343209.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_biobert_pipeline_en_4.3.0_3.2_1679309343209.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_biobert_pipeline", "en", "clinical/models") text = '''42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_biobert_pipeline", "en", "clinical/models") val text = "42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------|--------:|------:|:------------|-------------:| | 0 | woman | 12 | 16 | HUMAN | 0.9999 | | 1 | bacterial | 145 | 153 | SPECIES | 0.9981 | | 2 | Fusarium spp | 337 | 348 | SPECIES | 0.9873 | | 3 | patient | 355 | 361 | HUMAN | 0.9991 | | 4 | species | 507 | 513 | SPECIES | 0.9926 | | 5 | Fusarium solani complex | 522 | 544 | SPECIES | 0.8422 | | 6 | antifungals | 792 | 802 | SPECIES | 0.9929 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.3 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from iis2009002) author: John Snow Labs name: xlmroberta_ner_iis2009002_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `iis2009002`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_iis2009002_base_finetuned_panx_de_4.1.0_3.0_1660433851832.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_iis2009002_base_finetuned_panx_de_4.1.0_3.0_1660433851832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_iis2009002_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_iis2009002_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_iis2009002_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/iis2009002/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English BertForQuestionAnswering Cased model (from yossra) author: John Snow Labs name: bert_qa_yossra_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `yossra`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_yossra_finetuned_squad_en_4.0.0_3.0_1657186848644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_yossra_finetuned_squad_en_4.0.0_3.0_1657186848644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_yossra_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_yossra_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_yossra_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/yossra/bert-finetuned-squad --- layout: model title: English asr_wav2vec2_base_timit_demo_colab40 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab40 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab40` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab40_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab40_en_4.2.0_3.0_1664020850617.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab40_en_4.2.0_3.0_1664020850617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab40", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab40", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab40| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Finnish asr_wav2vec2_xlsr_train_aug_bigLM_1B TFWav2Vec2ForCTC from RASMUS author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B date: 2022-09-25 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_train_aug_bigLM_1B` is a Finnish model originally trained by RASMUS. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B_fi_4.2.0_3.0_1664097620804.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B_fi_4.2.0_3.0_1664097620804.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_train_aug_bigLM_1B| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|3.6 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Parties Clause Binary Classifier author: John Snow Labs name: legclf_parties_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `parties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `parties` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_parties_clause_en_1.0.0_3.2_1660123811373.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_parties_clause_en_1.0.0_3.2_1660123811373.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_parties_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[parties]| |[other]| |[other]| |[parties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_parties_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.97 0.96 91 parties 0.90 0.84 0.87 32 accuracy - - 0.93 123 macro-avg 0.92 0.91 0.91 123 weighted-avg 0.93 0.93 0.93 123 ``` --- layout: model title: English LongformerForQuestionAnswering model (from Nomi97) author: John Snow Labs name: longformer_qa_Chatbot date: 2022-06-26 tags: [en, open_source, longformer, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Chatbot_QA` is a English model originally trained by `Nomi97`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_qa_Chatbot_en_4.0.0_3.0_1656255131812.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_qa_Chatbot_en_4.0.0_3.0_1656255131812.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_qa_Chatbot","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_qa_Chatbot","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.longformer.by_Nomi97").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_qa_Chatbot| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|546.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Nomi97/Chatbot_QA --- layout: model title: Part of Speech for Arabic author: John Snow Labs name: pos_ud_padt date: 2021-03-09 tags: [part_of_speech, open_source, arabic, pos_ud_padt, ar] task: Part of Speech Tagging language: ar edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - X - VERB - NOUN - ADJ - ADP - PUNCT - NUM - None - PRON - SCONJ - CCONJ - DET - PART - ADV - SYM - AUX - PROPN - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_padt_ar_3.0.0_3.0_1615292251530.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_padt_ar_3.0.0_3.0_1615292251530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols("sentence") \ .setOutputCol("token") pos_tagger = PerceptronModel.pretrained("pos_ud_padt", "ar") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, pos_tagger ]) example = spark.createDataFrame([['مرحبا من جون سنو مختبرات! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val posTagger = PerceptronModel.pretrained("pos_ud_padt", "ar") .setInputCols("sentence", "token") .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, posTagger)) val data = Seq("مرحبا من جون سنو مختبرات! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""مرحبا من جون سنو مختبرات! ""] token_df = nlu.load('ar.pos').predict(text) token_df ```

## Results ```bash token pos 0 مرحبا NOUN 1 من ADP 2 جون X 3 سنو X 4 مختبرات NOUN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_padt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|ar| --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_small_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-small-finetuned-squad` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_finetuned_squad_en_4.0.0_3.0_1654184762850.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_small_finetuned_squad_en_4.0.0_3.0_1654184762850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_small_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_small_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.small.by_anas-awadalla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_small_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|107.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-small-finetuned-squad --- layout: model title: Translate Artificial languages to English Pipeline author: John Snow Labs name: translate_art_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, art, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `art` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_art_en_xx_2.7.0_2.4_1609686264952.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_art_en_xx_2.7.0_2.4_1609686264952.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_art_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_art_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.art.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_art_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Base Cased model (from mrm8488) author: John Snow Labs name: t5_base_finetuned_squadv2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-finetuned-squadv2` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_squadv2_en_4.3.0_3.0_1675109106503.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_squadv2_en_4.3.0_3.0_1675109106503.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_finetuned_squadv2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_finetuned_squadv2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_finetuned_squadv2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|861.2 MB| ## References - https://huggingface.co/mrm8488/t5-base-finetuned-squadv2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://rajpurkar.github.io/SQuAD-explorer/ - https://arxiv.org/pdf/1910.10683.pdf - https://i.imgur.com/jVFMMWR.png - https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: Spanish BertForMaskedLM Base Cased model (from dccuchile) author: John Snow Labs name: bert_embeddings_base_spanish_wwm_cased date: 2022-12-02 tags: [es, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased` is a Spanish model originally trained by `dccuchile`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_spanish_wwm_cased_es_4.2.4_3.0_1670018860888.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_spanish_wwm_cased_es_4.2.4_3.0_1670018860888.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_spanish_wwm_cased","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_spanish_wwm_cased","es") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_spanish_wwm_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|412.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased - https://github.com/google-research/bert - https://github.com/josecannete/spanish-corpora - https://github.com/google-research/bert/blob/master/multilingual.md - https://users.dcc.uchile.cl/~jperez/beto/uncased_2M/tensorflow_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/uncased_2M/pytorch_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/cased_2M/tensorflow_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/cased_2M/pytorch_weights.tar.gz - https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827 - https://www.kaggle.com/nltkdata/conll-corpora - https://github.com/gchaperon/beto-benchmarks/blob/master/conll2002/dev_results_beto-cased_conll2002.txt - https://github.com/facebookresearch/MLDoc - https://github.com/gchaperon/beto-benchmarks/blob/master/MLDoc/dev_results_beto-cased_mldoc.txt - https://github.com/gchaperon/beto-benchmarks/blob/master/MLDoc/dev_results_beto-uncased_mldoc.txt - https://github.com/google-research-datasets/paws/tree/master/pawsx - https://github.com/facebookresearch/XNLI - https://colab.research.google.com/drive/1uRwg4UmPgYIqGYY4gW_Nsw9782GFJbPt - https://www.adere.so/ - https://imfd.cl/en/ - https://www.tensorflow.org/tfrc - https://users.dcc.uchile.cl/~jperez/papers/pml4dc2020.pdf - https://github.com/google-research/bert/blob/master/multilingual.md - https://arxiv.org/pdf/1904.09077.pdf - https://arxiv.org/pdf/1906.01502.pdf - https://arxiv.org/abs/1812.10464 - https://arxiv.org/pdf/1901.07291.pdf - https://arxiv.org/pdf/1904.02099.pdf - https://arxiv.org/pdf/1906.01569.pdf - https://arxiv.org/abs/1908.11828 --- layout: model title: Detect Normalized Genes and Human Phenotypes author: John Snow Labs name: ner_human_phenotype_go_clinical date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect normalized mentions of genes (go) and human phenotypes (hp) in medical text. ## Predicted Entities : `GO`, `HP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HUMAN_PHENOTYPE_GO_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_clinical_en_3.0.0_3.0_1617209694955.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_clinical_en_3.0.0_3.0_1617209694955.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----+--------------------------+---------+-------+----------+ | | chunk | begin | end | entity | +====+==========================+=========+=======+==========+ | 0 | tumor | 39 | 43 | HP | +----+--------------------------+---------+-------+----------+ | 1 | tricarboxylic acid cycle | 79 | 102 | GO | +----+--------------------------+---------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_go_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|-----:|-----:|---------:|---------:|---------:| | 0 | B-GO | 1530 | 129 | 57 | 0.922242 | 0.964083 | 0.942699 | | 1 | B-HP | 950 | 133 | 130 | 0.877193 | 0.87963 | 0.87841 | | 2 | I-HP | 253 | 46 | 68 | 0.846154 | 0.788162 | 0.816129 | | 3 | I-GO | 4550 | 344 | 154 | 0.92971 | 0.967262 | 0.948114 | | 4 | Macro-average | 7283 | 652 | 409 | 0.893825 | 0.899784 | 0.896795 | | 5 | Micro-average | 7283 | 652 | 409 | 0.917832 | 0.946828 | 0.932105 | ``` --- layout: model title: English asr_wav2vec_large_xlsr_korean TFWav2Vec2ForCTC from fleek author: John Snow Labs name: pipeline_asr_wav2vec_large_xlsr_korean date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec_large_xlsr_korean` is a English model originally trained by fleek. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec_large_xlsr_korean_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec_large_xlsr_korean_en_4.2.0_3.0_1664098611361.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec_large_xlsr_korean_en_4.2.0_3.0_1664098611361.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec_large_xlsr_korean', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec_large_xlsr_korean", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec_large_xlsr_korean| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from Turkic Languages to English author: John Snow Labs name: opus_mt_trk_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, trk, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `trk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_trk_en_xx_2.7.0_2.4_1609167597007.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_trk_en_xx_2.7.0_2.4_1609167597007.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_trk_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from shaojie) author: John Snow Labs name: distilbert_qa_shaojie_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `shaojie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_shaojie_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772539534.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_shaojie_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772539534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shaojie_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shaojie_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_shaojie_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/shaojie/distilbert-base-uncased-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from lorenzkuhn) author: John Snow Labs name: roberta_qa_lorenzkuhn_base_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `lorenzkuhn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_lorenzkuhn_base_finetuned_squad_en_4.3.0_3.0_1674217419784.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_lorenzkuhn_base_finetuned_squad_en_4.3.0_3.0_1674217419784.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_lorenzkuhn_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_lorenzkuhn_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_lorenzkuhn_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/lorenzkuhn/roberta-base-finetuned-squad --- layout: model title: Summarize Radiology Reports author: John Snow Labs name: summarizer_radiology date: 2023-04-23 tags: [clinical, licensed, en, summarization, tensorflow, radiology] task: Summarization language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is capable of summarizing radiology reports while preserving the important information such as imaging tests and findings. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_radiology_en_4.4.0_3.0_1682218525772.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_radiology_en_4.4.0_3.0_1682218525772.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") summarizer = MedicalSummarizer.pretrained("summarizer_radiology", "en", "clinical/models")\ .setInputCols(["document"])\ .setOutputCol("summary")\ .setMaxTextLength(512)\ .setMaxNewTokens(512) pipeline = Pipeline(stages=[ document, summarizer ]) text = """INDICATIONS: Peripheral vascular disease with claudication. RIGHT: 1. Normal arterial imaging of right lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic. 4. Ankle brachial index is 0.96. LEFT: 1. Normal arterial imaging of left lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic throughout except in posterior tibial artery where it is biphasic. 4. Ankle brachial index is 1.06. IMPRESSION: Normal arterial imaging of both lower lobes. """ data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val summarizer = MedicalSummarizer.pretrained("summarizer_radiology", "en", "clinical/models") .setInputCols("document") .setOutputCol("summary") .setMaxTextLength(512) .setMaxNewTokens(512) val pipeline = new Pipeline().setStages(Array(document_assembler, summarizer)) val text = """INDICATIONS: Peripheral vascular disease with claudication. RIGHT: 1. Normal arterial imaging of right lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic. 4. Ankle brachial index is 0.96. LEFT: 1. Normal arterial imaging of left lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic throughout except in posterior tibial artery where it is biphasic. 4. Ankle brachial index is 1.06. IMPRESSION: Normal arterial imaging of both lower lobes. """ val data = Seq(text).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash The patient has peripheral vascular disease with claudication. The right lower extremity shows normal arterial imaging, but the peak systolic velocity is normal. The arterial waveform is triphasic throughout, except for the posterior tibial artery, which is biphasic. The ankle brachial index is 0.96. The impression is normal arterial imaging of both lower lobes. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_radiology| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.4 MB| --- layout: model title: English BertForQuestionAnswering model (from AnonymousSub) author: John Snow Labs name: bert_qa_news_pretrain_bert_FT_new_newsqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `news_pretrain_bert_FT_new_newsqa` is a English model orginally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_news_pretrain_bert_FT_new_newsqa_en_4.0.0_3.0_1654188909680.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_news_pretrain_bert_FT_new_newsqa_en_4.0.0_3.0_1654188909680.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_news_pretrain_bert_FT_new_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_news_pretrain_bert_FT_new_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.bert.new.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_news_pretrain_bert_FT_new_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/news_pretrain_bert_FT_new_newsqa --- layout: model title: Modern Greek (1453-) asr_greek_lsr_1 TFWav2Vec2ForCTC from skylord author: John Snow Labs name: asr_greek_lsr_1 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_greek_lsr_1` is a Modern Greek (1453-) model originally trained by skylord. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_greek_lsr_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_greek_lsr_1_el_4.2.0_3.0_1664110738611.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_greek_lsr_1_el_4.2.0_3.0_1664110738611.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_greek_lsr_1", "el")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_greek_lsr_1", "el") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_greek_lsr_1| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|el| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering model (from healx) author: John Snow Labs name: bert_qa_biomedical_slot_filling_reader_base date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biomedical-slot-filling-reader-base` is a English model orginally trained by `healx`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biomedical_slot_filling_reader_base_en_4.0.0_3.0_1654185786674.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biomedical_slot_filling_reader_base_en_4.0.0_3.0_1654185786674.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biomedical_slot_filling_reader_base","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biomedical_slot_filling_reader_base","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bio_medical.bert.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biomedical_slot_filling_reader_base| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/healx/biomedical-slot-filling-reader-base - https://arxiv.org/abs/2109.08564 --- layout: model title: Legal Dissolution Clause Binary Classifier author: John Snow Labs name: legclf_dissolution_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `dissolution` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `dissolution` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_dissolution_clause_en_1.0.0_3.2_1660122374357.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_dissolution_clause_en_1.0.0_3.2_1660122374357.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_dissolution_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[dissolution]| |[other]| |[other]| |[dissolution]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_dissolution_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support dissolution 0.95 0.90 0.92 39 other 0.97 0.99 0.98 137 accuracy - - 0.97 176 macro-avg 0.96 0.94 0.95 176 weighted-avg 0.97 0.97 0.97 176 ``` --- layout: model title: Clinical Portuguese Bert Embeddiongs (Biomedical) author: John Snow Labs name: biobert_embeddings_biomedical date: 2022-04-11 tags: [biobert, embeddings, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BioBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `biobertpt-bio` is a Portuguese model orginally trained by `pucpr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_embeddings_biomedical_pt_3.4.2_3.0_1649687586887.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_embeddings_biomedical_pt_3.4.2_3.0_1649687586887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_embeddings_biomedical| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|667.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/pucpr/biobertpt-bio - https://aclanthology.org/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: Adverse Drug Events Classifier (BERT) author: John Snow Labs name: bert_sequence_classifier_ade date: 2022-02-08 tags: [bert, sequence_classification, en, licensed] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify texts/sentences in two categories: - `True` : The sentence is talking about a possible ADE. - `False` : The sentence doesn’t have any information about an ADE. This model is a [BioBERT-based](https://github.com/dmis-lab/biobert) classifier. ## Predicted Entities `True`, `False` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_ADE/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/08.3.MedicalBertForSequenceClassification_in_SparkNLP.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_ade_en_3.4.1_3.0_1644324436716.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_ade_en_3.4.1_3.0_1644324436716.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------+------+ |text |result| +----------------------------------------------------------------+------+ |I felt a bit drowsy and had blurred vision after taking Aspirin.|[True]| +----------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_ade| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model is trained on a custom dataset comprising of CADEC, DRUG-AE and Twimed. ## Benchmarking ```bash label precision recall f1-score support False 0.97 0.97 0.97 6884 True 0.87 0.85 0.86 1398 accuracy 0.95 0.95 0.95 8282 macro-avg 0.92 0.91 0.91 8282 weighted-avg 0.95 0.95 0.95 8282 ``` --- layout: model title: English RobertaForQuestionAnswering (from AyushPJ) author: John Snow Labs name: roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ai-club-inductions-21-nlp-roBERTa-base-squad-v2` is a English model originally trained by `AyushPJ`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2_en_4.0.0_3.0_1655727560873.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2_en_4.0.0_3.0_1655727560873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_v2.by_AyushPJ").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ai_club_inductions_21_nlp_roBERTa_base_squad_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|465.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AyushPJ/ai-club-inductions-21-nlp-roBERTa-base-squad-v2 --- layout: model title: Legal Listing Clause Binary Classifier author: John Snow Labs name: legclf_listing_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `listing` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `listing` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_listing_clause_en_1.0.0_3.2_1660123698313.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_listing_clause_en_1.0.0_3.2_1660123698313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_listing_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[listing]| |[other]| |[other]| |[listing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_listing_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support listing 1.00 0.97 0.99 38 other 0.99 1.00 0.99 92 accuracy - - 0.99 130 macro-avg 0.99 0.99 0.99 130 weighted-avg 0.99 0.99 0.99 130 ``` --- layout: model title: English Named Entity Recognition (from lucifermorninstar011) author: John Snow Labs name: distilbert_ner_autotrain_lucifer_morningstar_job_859227344 date: 2022-05-16 tags: [distilbert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-lucifer_morningstar_job-859227344` is a English model orginally trained by `lucifermorninstar011`. ## Predicted Entities `Job`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_autotrain_lucifer_morningstar_job_859227344_en_3.4.2_3.0_1652721635851.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_autotrain_lucifer_morningstar_job_859227344_en_3.4.2_3.0_1652721635851.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_autotrain_lucifer_morningstar_job_859227344| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/lucifermorninstar011/autotrain-lucifer_morningstar_job-859227344 --- layout: model title: Spanish DistilBertForQuestionAnswering model (from CenIA) MLQA author: John Snow Labs name: distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa date: 2022-06-08 tags: [es, open_source, distilbert, question_answering] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distillbert-base-spanish-uncased-finetuned-qa-mlqa` is a English model originally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa_es_4.0.0_3.0_1654728089558.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa_es_4.0.0_3.0_1654728089558.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa","es") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.mlqa.distil_bert.base_uncased").predict("""¿Cuál es mi nombre?|||"Mi nombre es Clara y vivo en Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distillbert_base_spanish_uncased_finetuned_qa_mlqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|250.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/distillbert-base-spanish-uncased-finetuned-qa-mlqa --- layout: model title: English Bert Embeddings Cased model (from aditeyabaral) author: John Snow Labs name: bert_embeddings_carlbert_webex_mlm_spatial date: 2023-02-22 tags: [open_source, bert, bert_embeddings, bertformaskedlm, en, tensorflow] task: Embeddings language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `carlbert-webex-mlm-spatial` is a English model originally trained by `aditeyabaral`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_carlbert_webex_mlm_spatial_en_4.3.0_3.0_1677087512961.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_carlbert_webex_mlm_spatial_en_4.3.0_3.0_1677087512961.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_carlbert_webex_mlm_spatial","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["Ich liebe Spark-NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_carlbert_webex_mlm_spatial","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("Ich liebe Spark-NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_carlbert_webex_mlm_spatial| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|406.6 MB| |Case sensitive:|true| ## References https://huggingface.co/aditeyabaral/carlbert-webex-mlm-spatial --- layout: model title: Stopwords Remover for Swedish language (386 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, sv, open_source] task: Stop Words Removal language: sv edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_sv_3.4.1_3.0_1646672982973.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_sv_3.4.1_3.0_1646672982973.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[är] | +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|sv| |Size:|2.5 KB| --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Original_BioBERT_512 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Original-BioBERT-512` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_BioBERT_512_en_4.0.0_3.0_1657108546349.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_BioBERT_512_en_4.0.0_3.0_1657108546349.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Original_BioBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Original-BioBERT-512 --- layout: model title: English image_classifier_vit_trainer_rare_puppers ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_trainer_rare_puppers date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_trainer_rare_puppers` is a English model originally trained by nateraw. ## Predicted Entities `corgi`, `samoyed`, `shiba inu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_trainer_rare_puppers_en_4.1.0_3.0_1660169700721.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_trainer_rare_puppers_en_4.1.0_3.0_1660169700721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_trainer_rare_puppers", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_trainer_rare_puppers", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_trainer_rare_puppers| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect PHI (Deidentification) author: John Snow Labs name: ner_deid_large_pipeline date: 2023-03-13 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_large](https://nlp.johnsnowlabs.com/2021/03/31/ner_deid_large_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_large_pipeline_en_4.3.0_3.2_1678736100712.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_large_pipeline_en_4.3.0_3.2_1678736100712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_large_pipeline", "en", "clinical/models") text = '''HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_large_pipeline", "en", "clinical/models") val text = "HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid_large.pipeline").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------|--------:|------:|:------------|-------------:| | 0 | Smith | 32 | 36 | NAME | 0.9998 | | 1 | VA Hospital | 184 | 194 | LOCATION | 0.68335 | | 2 | Day Hospital | 258 | 269 | LOCATION | 0.7763 | | 3 | 02/04/2003 | 341 | 350 | DATE | 1 | | 4 | Smith | 374 | 378 | NAME | 0.9993 | | 5 | Day Hospital | 397 | 408 | LOCATION | 0.7522 | | 6 | Smith | 782 | 786 | NAME | 0.9998 | | 7 | Smith | 1131 | 1135 | NAME | 0.9997 | | 8 | 7 Ardmore Tower | 1153 | 1167 | LOCATION | 0.739867 | | 9 | Hart | 1221 | 1224 | NAME | 0.9995 | | 10 | Smith | 1231 | 1235 | NAME | 0.9998 | | 11 | 02/07/2003 | 1329 | 1338 | DATE | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RoBERTa Embeddings (Sampling strategy 'sim select') author: John Snow Labs name: roberta_embeddings_distilroberta_base_climate_s date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilroberta-base-climate-s` is a English model orginally trained by `climatebert`. Sampling strategy s:As expressed in the author's paper [here](https://arxiv.org/pdf/2110.12010.pdf), s is "sim select", meaning 70% of the most similar sentences of one of the corpus was used, discarding the rest. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_s_en_3.4.2_3.0_1649946847931.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_s_en_3.4.2_3.0_1649946847931.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_distilroberta_base_climate_s| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|310.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/climatebert/distilroberta-base-climate-s - https://arxiv.org/abs/2110.12010 --- layout: model title: Stop Words Cleaner for Latvian author: John Snow Labs name: stopwords_lv date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: lv edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, lv] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_lv_lv_2.5.4_2.4_1594742439893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_lv_lv_2.5.4_2.4_1594742439893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_lv", "lv") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Džons Snovs ir ne tikai ziemeļu karalis, bet arī angļu ārsts un anestēzijas un medicīniskās higiēnas attīstības līderis.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_lv", "lv") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Džons Snovs ir ne tikai ziemeļu karalis, bet arī angļu ārsts un anestēzijas un medicīniskās higiēnas attīstības līderis.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Džons Snovs ir ne tikai ziemeļu karalis, bet arī angļu ārsts un anestēzijas un medicīniskās higiēnas attīstības līderis."""] stopword_df = nlu.load('lv.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Džons', metadata={'sentence': '0'}), Row(annotatorType='token', begin=6, end=10, result='Snovs', metadata={'sentence': '0'}), Row(annotatorType='token', begin=24, end=30, result='ziemeļu', metadata={'sentence': '0'}), Row(annotatorType='token', begin=32, end=38, result='karalis', metadata={'sentence': '0'}), Row(annotatorType='token', begin=39, end=39, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_lv| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|lv| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: French RobertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: roberta_qa_ADDI_FR_XLM_R date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: fr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-FR-XLM-R` is a French model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FR_XLM_R_fr_4.0.0_3.0_1655726482123.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FR_XLM_R_fr_4.0.0_3.0_1655726482123.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ADDI_FR_XLM_R","fr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ADDI_FR_XLM_R","fr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fr.answer_question.xlm_roberta.fr_tuned.by_Gantenbein").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ADDI_FR_XLM_R| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fr| |Size:|422.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-FR-XLM-R --- layout: model title: Relation Extraction Between Body Parts and Procedures author: John Snow Labs name: redl_bodypart_procedure_test_biobert date: 2023-01-14 tags: [relation_extraction, en, clinical, dl, licensed, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like ‘Internal_organ_or_component’, ’External_body_part_or_region’ etc. and procedure and test entities. 1 : body part and test/procedure are related to each other. 0 : body part and test/procedure are not related to each other. ## Predicted Entities `1`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_4.2.4_3.0_1673714088228.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_4.2.4_3.0_1673714088228.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | chunk1 | entity2 | chunk2 | confidence | |---:|-----------:|:-----------------------------|:---------|:----------|:--------------------|-------------:| | 0 | 1 | External_body_part_or_region | chest | Test | portable ultrasound | 0.99953 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_bodypart_procedure_test_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References Trained on a custom internal dataset. ## Benchmarking ```bash label Recall Precision F1 Support 0 0.338 0.472 0.394 325 1 0.904 0.843 0.872 1275 Avg. 0.621 0.657 0.633 - ``` --- layout: model title: Translate Morisyen to English Pipeline author: John Snow Labs name: translate_mfe_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, mfe, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `mfe` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_mfe_en_xx_2.7.0_2.4_1609688095194.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_mfe_en_xx_2.7.0_2.4_1609688095194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_mfe_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_mfe_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.mfe.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_mfe_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from haesun) author: John Snow Labs name: xlmroberta_ner_haesun_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `haesun`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_haesun_base_finetuned_panx_all_xx_4.1.0_3.0_1660428351266.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_haesun_base_finetuned_panx_all_xx_4.1.0_3.0_1660428351266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_haesun_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_haesun_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_haesun_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|862.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/haesun/xlm-roberta-base-finetuned-panx-all --- layout: model title: Ocr pipeline with Rest-Api author: John Snow Labs name: ocr_restapi date: 2023-01-03 tags: [en, licensed, ocr, RestApi] task: Ocr RestApi language: en nav_key: models edition: Visual NLP 4.0.0 spark_version: 3.2.1 supported: true annotator: OcrRestApi article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description RestAPI pipeline implementation for the OCR task, using tesseract models. Tesseract is an Optical Character Recognition (OCR) engine developed by Google. It is an open-source tool that can be used to recognize text in images and convert it into machine-readable text. The engine is based on a neural network architecture and uses machine learning algorithms to improve its accuracy over time. Tesseract has been trained on a variety of datasets to improve its recognition capabilities. These datasets include images of text in various languages and scripts, as well as images with different font styles, sizes, and orientations. The training process involves feeding the engine with a large number of images and their corresponding text, allowing the engine to learn the patterns and characteristics of different text styles. One of the most important datasets used in training Tesseract is the UNLV dataset, which contains over 400,000 images of text in different languages, scripts, and font styles. This dataset is widely used in the OCR community and has been instrumental in improving the accuracy of Tesseract. Other datasets that have been used in training Tesseract include the ICDAR dataset, the IIIT-HWS dataset, and the RRC-GV-WS dataset. In addition to these datasets, Tesseract also uses a technique called adaptive training, where the engine can continuously improve its recognition capabilities by learning from new images and text. This allows Tesseract to adapt to new text styles and languages, and improve its overall accuracy. ## Predicted Entities {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/blob/master/tutorials/Certification_Trainings/6.2.SparkOcrRestApi.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) ocr = ImageToText() \ .setInputCol("image") \ .setOutputCol("text") pipeline = PipelineModel(stages=[ binary_to_image, ocr ]) ## Start server SERVER_HOST = "localhost" SERVER_PORT = 8889 SERVER_API_NAME = "spark_ocr_api" checkpoint_dir = tempfile.TemporaryDirectory("_spark_ocr_server_checkpoint") df = spark.readStream.server() \ .address(SERVER_HOST, SERVER_PORT, SERVER_API_NAME) \ .load() \ .parseRequest(SERVER_API_NAME, schema=StructType().add("image", BinaryType())) \ .withColumn("path", f.lit("")) \ .withColumnRenamed("image", "content") replies = pipeline.transform(df)\ .makeReply("text") server = replies\ .writeStream \ .server() \ .replyTo(SERVER_API_NAME) \ .queryName("spark_ocr") \ .option("checkpointLocation", checkpoint_dir) \ .start() ## Call API imagePath = pkg_resources.resource_filename('sparkocr', '/resources/ocr/images/check.jpg') with open(imagePath, "rb") as image_file: im_bytes = image_file.read() im_b64 = base64.b64encode(im_bytes).decode("utf8") headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} payload = json.dumps({"image": im_b64}) r = requests.post(data=payload, headers=headers, url=f"http://{SERVER_HOST}:{SERVER_PORT}/{SERVER_API_NAME}") ```

## Example ### Input: ![Screenshot](/assets/images/examples_ocr/image2.png) ## Output text ```bash Response: STARBUCKS Store #19208 11902 Euclid Avenue Cleveland, OH (216) 229-U749 CHK 664250 12/07/2014 06:43 PM 112003. Drawers 2. Reg: 2 ¥t Pep Mocha 4.5 Sbux Card 495 AMXARKERARANG 228 Subtotal $4.95 Total $4.95 Change Cue BO LOO - Check Closed ~ "49/07/2014 06:43 py oBUX Card «3228 New Balance: 37.45 Card is registertd ``` ## Model Information {:.table-model} |---|---| |Model Name:|ocr_restapi| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Pipeline to Extract Mentions of Response to Cancer Treatment author: John Snow Labs name: ner_oncology_response_to_treatment_pipeline date: 2023-03-09 tags: [licensed, clinical, en, oncology, ner, treatment] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_response_to_treatment](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_response_to_treatment_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_pipeline_en_4.3.0_3.2_1678349824229.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_pipeline_en_4.3.0_3.2_1678349824229.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_response_to_treatment_pipeline", "en", "clinical/models") text = '''She completed her first-line therapy, but some months later there was recurrence of the breast cancer.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_response_to_treatment_pipeline", "en", "clinical/models") val text = "She completed her first-line therapy, but some months later there was recurrence of the breast cancer." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:----------------------|-------------:| | 0 | recurrence | 70 | 79 | Response_To_Treatment | 0.9767 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_response_to_treatment_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering Cased model (from skandaonsolve) author: John Snow Labs name: roberta_qa_finetuned_timeentities date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-timeentities` is a English model originally trained by `skandaonsolve`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities_en_4.3.0_3.0_1674220613032.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_timeentities_en_4.3.0_3.0_1674220613032.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_timeentities","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_timeentities| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-timeentities --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_tiny_2_finetuned_squadv2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-2-finetuned-squadv2` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_tiny_2_finetuned_squadv2_en_4.0.0_3.0_1654184806087.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_tiny_2_finetuned_squadv2_en_4.0.0_3.0_1654184806087.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_tiny_2_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_tiny_2_finetuned_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.tiny_v2.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_tiny_2_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|19.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-tiny-2-finetuned-squadv2 --- layout: model title: Part of Speech for Icelandic author: John Snow Labs name: pos_icepahc date: 2021-03-23 tags: [pos, open_source, is] supported: true task: Part of Speech Tagging language: is edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_icepahc_is_2.7.5_2.4_1616509019245.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_icepahc_is_2.7.5_2.4_1616509019245.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_icepahc", "is")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Númerið blikkaði á skjánum eins og einmana vekjaraklukka um nótt á níundu hæð í gamalli blokk í austurbæ Reykjavíkur .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_icepahc", "is") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("Númerið blikkaði á skjánum eins og einmana vekjaraklukka um nótt á níundu hæð í gamalli blokk í austurbæ Reykjavíkur .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Númerið blikkaði á skjánum eins og einmana vekjaraklukka um nótt á níundu hæð í gamalli blokk í austurbæ Reykjavíkur .""] token_df = nlu.load('is.pos.icepahc').predict(text) token_df ```

## Results ```bash +----------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ |text |result | +----------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ |Númerið blikkaði á skjánum eins og einmana vekjaraklukka um nótt á níundu hæð í gamalli blokk í austurbæ Reykjavíkur .|[NOUN, VERB, ADP, NOUN, ADV, ADP, ADJ, NOUN, ADP, NOUN, ADP, ADJ, NOUN, ADP, ADJ, NOUN, ADP, PROPN, PROPN, PUNCT]| +----------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_icepahc| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|is| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.81 | 0.74 | 0.78 | 5906 | | ADP | 0.95 | 0.96 | 0.96 | 15548 | | ADV | 0.90 | 0.90 | 0.90 | 10631 | | AUX | 0.92 | 0.93 | 0.92 | 7416 | | CCONJ | 0.96 | 0.97 | 0.96 | 8437 | | DET | 0.89 | 0.87 | 0.88 | 7476 | | INTJ | 0.95 | 0.77 | 0.85 | 131 | | NOUN | 0.90 | 0.92 | 0.91 | 20726 | | NUM | 0.75 | 0.83 | 0.79 | 655 | | PART | 0.96 | 0.96 | 0.96 | 1703 | | PRON | 0.94 | 0.96 | 0.95 | 16852 | | PROPN | 0.89 | 0.89 | 0.89 | 4444 | | PUNCT | 0.98 | 0.98 | 0.98 | 16434 | | SCONJ | 0.94 | 0.94 | 0.94 | 5663 | | VERB | 0.92 | 0.90 | 0.91 | 17329 | | X | 0.60 | 0.30 | 0.40 | 346 | | accuracy | | | 0.92 | 139697 | | macro avg | 0.89 | 0.86 | 0.87 | 139697 | | weighted avg | 0.92 | 0.92 | 0.92 | 139697 | ``` --- layout: model title: Part of Speech for Marathi author: John Snow Labs name: pos_ud_ufal date: 2021-03-09 tags: [part_of_speech, open_source, marathi, pos_ud_ufal, mr] task: Part of Speech Tagging language: mr edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - DET - AUX - NOUN - PUNCT - PRON - ADJ - CCONJ - ADV - VERB - SCONJ - NUM - ADP - INTJ - PROPN {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_ufal_mr_3.0.0_3.0_1615292224912.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_ufal_mr_3.0.0_3.0_1615292224912.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_ufal", "mr") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['जॉन हिम लॅब्समधून हॅलो! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_ufal", "mr") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("जॉन हिम लॅब्समधून हॅलो! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""जॉन हिम लॅब्समधून हॅलो! ""] token_df = nlu.load('mr.pos').predict(text) token_df ```

## Results ```bash token pos 0 जॉन PROPN 1 हिम NOUN 2 लॅब्समधून ADJ 3 हॅलो VERB 4 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_ufal| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|mr| --- layout: model title: Japanese Word Segmentation author: John Snow Labs name: wordseg_gsd_ud date: 2021-01-03 task: Word Segmentation language: ja edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [word_segmentation, ja, open_source] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know _a priori_ which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. References: - Xue, Nianwen. "Chinese word segmentation as character tagging." International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_ja_2.7.0_2.4_1609692613721.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_ja_2.7.0_2.4_1609692613721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline as a substitute of the Tokenizer stage.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.pretrained('wordseg_gsd_ud', 'ja')\ .setInputCols("document")\ .setOutputCol("token") pipeline = Pipeline(stages=[ document_assembler, word_segmenter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['清代は湖北省が置かれ、そのまま現代の行政区分になっている。']], ["text"]) result = model.transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") .setInputCols("document") .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("清代は湖北省が置かれ、そのまま現代の行政区分になっている。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""清代は湖北省が置かれ、そのまま現代の行政区分になっている。"""] token_df = nlu.load('ja.segment_words').predict(text, output_level='token') token_df ```

## Results ```bash +----------------------------------------------------------+------------------------------------------------------------------------------------------------+ |text |result | +----------------------------------------------------------+------------------------------------------------------------------------------------------------+ |清代は湖北省が置かれ、そのまま現代の行政区分になっている。|[清代, は, 湖北, 省, が, 置か, れ, 、, その, まま, 現代, の, 行政, 区分, に, なっ, て, いる, 。]| +----------------------------------------------------------+------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_gsd_ud| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|ja| ## Data Source We trained this model on the the [Universal Dependenicies](universaldependencies.org) data set from Google (GSD-UD). > Asahara, M., Kanayama, H., Tanaka, T., Miyao, Y., Uematsu, S., Mori, S., Matsumoto, Y., Omura, M., & Murawaki, Y. (2018). Universal Dependencies Version 2 for Japanese. In LREC-2018. ## Benchmarking ```bash | Model | precision | recall | f1-score | |---------------|--------------|--------------|--------------| | JA_UD-GSD | 0,7687 | 0,8048 | 0,7863 | ``` --- layout: model title: English BertForTokenClassification Cased model (from test123) author: John Snow Labs name: bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765 date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-ingredient_pseudo_label_training_ner-29576765` is a English model originally trained by `test123`. ## Predicted Entities `I`, `B` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765_en_4.2.4_3.0_1669814198805.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765_en_4.2.4_3.0_1669814198805.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autonlp_ingredient_pseudo_label_training_ner_29576765| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/test123/autonlp-ingredient_pseudo_label_training_ner-29576765 --- layout: model title: Legal NER for NDA (Confidential Information-Permissions) author: John Snow Labs name: legner_nda_confidential_information_permissions date: 2023-04-06 tags: [en, licensed, legal, ner, nda, permission] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `USE_OF_CONF_INFO ` clause with a proper classifier (use legmulticlf_mnda_sections_paragraph_other for that purpose). It will extract the following entities: `PERMISSION`, `PERMISSION_SUBJECT `, `PERMISSION_OBJECT `, and `PERMISSION_IND_OBJECT `. ## Predicted Entities `PERMISSION`, `PERMISSION_SUBJECT`, `PERMISSION_OBJECT`, `PERMISSION_IND_OBJECT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_confidential_information_permissions_en_1.0.0_3.0_1680814300223.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_confidential_information_permissions_en_1.0.0_3.0_1680814300223.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_confidential_information_permissions", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""The interested party may disclose the information to its financing sources and potential financing sources provided that such financing sources are bound by the terms of this non-disclosure agreement and agree to keep the information confidential."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +---------------------------+---------------------+ |chunk |ner_label | +---------------------------+---------------------+ |interested party |PERMISSION_SUBJECT | |disclose |PERMISSION | |information |PERMISSION_OBJECT | |financing sources |PERMISSION_IND_OBJECT| |potential financing sources|PERMISSION_IND_OBJECT| +---------------------------+---------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_confidential_information_permissions| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support PERMISSION 1.00 1.00 1.00 9 PERMISSION_IND_OBJECT 1.00 0.67 0.80 9 PERMISSION_OBJECT 0.91 1.00 0.95 10 PERMISSION_SUBJECT 0.90 1.00 0.95 9 micro-avg 0.94 0.92 0.93 37 macro-avg 0.95 0.92 0.92 37 weighted-avg 0.95 0.92 0.93 37 ``` --- layout: model title: English RobertaForQuestionAnswering (from nlpconnect) author: John Snow Labs name: roberta_qa_dpr_nq_reader_roberta_base_v2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dpr-nq-reader-roberta-base-v2` is a English model originally trained by `nlpconnect`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_dpr_nq_reader_roberta_base_v2_en_4.0.0_3.0_1655728533632.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_dpr_nq_reader_roberta_base_v2_en_4.0.0_3.0_1655728533632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_dpr_nq_reader_roberta_base_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_dpr_nq_reader_roberta_base_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.base_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_dpr_nq_reader_roberta_base_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpconnect/dpr-nq-reader-roberta-base-v2 --- layout: model title: Translate Germanic languages to English Pipeline author: John Snow Labs name: translate_gem_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, gem, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `gem` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_gem_en_xx_2.7.0_2.4_1609691653097.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_gem_en_xx_2.7.0_2.4_1609691653097.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gem_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gem_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gem.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gem_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForTokenClassification Base Uncased model (from Datasaur) author: John Snow Labs name: distilbert_token_classifier_base_uncased_finetuned_conll2003 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-conll2003` is a English model originally trained by `Datasaur`. ## Predicted Entities `PER`, `ORG`, `MISC`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_uncased_finetuned_conll2003_en_4.3.1_3.0_1678783094748.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_uncased_finetuned_conll2003_en_4.3.1_3.0_1678783094748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_finetuned_conll2003","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_finetuned_conll2003","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_base_uncased_finetuned_conll2003| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Datasaur/distilbert-base-uncased-finetuned-conll2003 --- layout: model title: English asr_xlsr_punctuation TFWav2Vec2ForCTC from boris author: John Snow Labs name: pipeline_asr_xlsr_punctuation date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_punctuation` is a English model originally trained by boris. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xlsr_punctuation_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_punctuation_en_4.2.0_3.0_1664020787424.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_punctuation_en_4.2.0_3.0_1664020787424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xlsr_punctuation', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xlsr_punctuation", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xlsr_punctuation| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Part of Speech for Portuguese author: John Snow Labs name: pos_ud_bosque date: 2020-05-03 12:54:00 +0800 task: Part of Speech Tagging language: pt edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, pt] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bosque_pt_2.5.0_2.4_1588499443093.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bosque_pt_2.5.0_2.4_1588499443093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_bosque", "pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_bosque", "pt") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica."""] pos_df = nlu.load('pt.pos.ud_bosque').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=3, result='ADV', metadata={'word': 'Além'}), Row(annotatorType='pos', begin=5, end=6, result='ADP', metadata={'word': 'de'}), Row(annotatorType='pos', begin=8, end=10, result='AUX', metadata={'word': 'ser'}), Row(annotatorType='pos', begin=12, end=12, result='DET', metadata={'word': 'o'}), Row(annotatorType='pos', begin=14, end=16, result='NOUN', metadata={'word': 'rei'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bosque| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|pt| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English RobertaForSequenceClassification Cased model (from lucianpopa) author: John Snow Labs name: roberta_classifier_autonlp_sst1_529214890 date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-SST1-529214890` is a English model originally trained by `lucianpopa`. ## Predicted Entities `1`, `0`, `4`, `3`, `2` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_autonlp_sst1_529214890_en_4.2.4_3.0_1670622060169.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_autonlp_sst1_529214890_en_4.2.4_3.0_1670622060169.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autonlp_sst1_529214890","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autonlp_sst1_529214890","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_autonlp_sst1_529214890| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|435.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/lucianpopa/autonlp-SST1-529214890 --- layout: model title: Smaller BERT Sentence Embeddings (L-10_H-128_A-2) author: John Snow Labs name: sent_small_bert_L10_128 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L10_128_en_2.6.0_2.4_1598350346103.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L10_128_en_2.6.0_2.4_1598350346103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_128", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_128", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L10_128').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L10_128_embeddings sentence [-0.3761860430240631, -0.04432673007249832, 0.... I hate cancer [-0.17762605845928192, -0.7492673397064209, -0... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L10_128| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|128| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1](https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: roberta_qa_deepset_base_squad2_orkg_what_1e_4 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deepset-roberta-base-squad2-orkg-what-1e-4` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_what_1e_4_en_4.3.0_3.0_1674209722792.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_what_1e_4_en_4.3.0_3.0_1674209722792.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_what_1e_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_what_1e_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_base_squad2_orkg_what_1e_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Moussab/deepset-roberta-base-squad2-orkg-what-1e-4 --- layout: model title: HCP Consult Classification Pipeline - Voice of the Patient author: John Snow Labs name: bert_sequence_classifier_vop_hcp_consult_pipeline date: 2023-06-14 tags: [licensed, en, clinical, classification, vop] task: Text Classification language: en edition: Healthcare NLP 4.4.3 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline includes the Medical Bert for Sequence Classification model to identify texts that mention a HCP consult. The pipeline is built on the top of [bert_sequence_classifier_vop_hcp_consult](https://nlp.johnsnowlabs.com/2023/06/13/bert_sequence_classifier_vop_hcp_consult_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_hcp_consult_pipeline_en_4.4.3_3.2_1686708308086.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_hcp_consult_pipeline_en_4.4.3_3.2_1686708308086.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | text | prediction | |:-----------------------------------------------------------------------------------------------------------------------|:-----------------| | My son has been to two doctors who gave him antibiotic drops but they also say the problem might related to allergies. | Consulted_By_HCP | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_hcp_consult_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|406.4 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForSequenceClassification --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8_en_4.0.0_3.0_1655732330256.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8_en_4.0.0_3.0_1655732330256.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_256d_seed_8").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|426.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-8 --- layout: model title: Pipeline to Detect diseases in Medical Text (biobert) author: John Snow Labs name: ner_diseases_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, disease, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_diseases_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_diseases_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_pipeline_en_3.4.1_3.0_1647871907471.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_pipeline_en_3.4.1_3.0_1647871907471.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_diseases_biobert_pipeline", "en", "clinical/models") pipeline.annotate("""Detection of various other intracellular signaling proteins is also described. Multiple autoimmune syndrome has been detected. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. She has Chikungunya virus disease story also. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ``` ```scala val pipeline = new PretrainedPipeline("ner_diseases_biobert_pipeline", "en", "clinical/models") pipeline.annotate("Detection of various other intracellular signaling proteins is also described. Multiple autoimmune syndrome has been detected. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. She has Chikungunya virus disease story also. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.diseases_biobert.pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Multiple autoimmune syndrome has been detected. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. She has Chikungunya virus disease story also. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash +-------------------------+---------+ |chunk |ner_label| +-------------------------+---------+ |autoimmune syndrome |Disease | |human T-cell leukemia |Disease | |T-cell leukemia |Disease | |Chikungunya virus disease|Disease | +-------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_diseases_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Detect PHI for Generic Deidentification in Romanian (BERT) author: John Snow Labs name: ner_deid_generic_bert date: 2022-07-06 tags: [deidentification, bert, phi, ner, generic, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 3.5.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It is trained with bert_base_cased embeddings and can detect 7 generic entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_3.5.0_3.0_1657112906624.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_3.5.0_3.0_1657112906624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|LOCATION | |Drumul Oprea Nr |LOCATION | |972 |LOCATION | |Vaslui |LOCATION | |737405 |LOCATION | |+40(235)413773 |CONTACT | |25 May 2022 |DATE | |BUREAN MARIA |NAME | |77 |AGE | |Agota Evelyn Tımar |NAME | |2450502264401 |ID | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_bert| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.5 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.95 0.97 0.96 1186 CONTACT 0.99 0.98 0.98 366 DATE 0.96 0.92 0.94 4518 ID 1.00 1.00 1.00 679 LOCATION 0.91 0.90 0.90 1683 NAME 0.93 0.96 0.94 2916 PROFESSION 0.87 0.85 0.86 161 micro-avg 0.94 0.94 0.94 11509 macro-avg 0.94 0.94 0.94 11509 weighted-avg 0.95 0.94 0.94 11509 ``` --- layout: model title: Part of Speech for Chinese author: John Snow Labs name: pos_ud_gsd date: 2020-05-04 20:11:00 +0800 task: Part of Speech Tagging language: zh edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, zh] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_zh_2.5.0_2.4_1588611712161.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_zh_2.5.0_2.4_1588611712161.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_gsd", "zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_gsd", "zh") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。"""] pos_df = nlu.load('zh.pos.ud_gsd').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=20, result='NOUN', metadata={'word': '除了担任北方国王之外，约翰·斯诺（John'}), Row(annotatorType='pos', begin=22, end=50, result='X', metadata={'word': 'Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|zh| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English DistilBertForQuestionAnswering model (from Slavka) author: John Snow Labs name: distilbert_qa_distil_bert_finetuned_log_parser_1 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distil-bert-finetuned-log-parser-1` is a English model originally trained by `Slavka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_distil_bert_finetuned_log_parser_1_en_4.0.0_3.0_1654723459354.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_distil_bert_finetuned_log_parser_1_en_4.0.0_3.0_1654723459354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distil_bert_finetuned_log_parser_1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distil_bert_finetuned_log_parser_1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.log_parser.by_Slavka").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distil_bert_finetuned_log_parser_1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Slavka/distil-bert-finetuned-log-parser-1 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from Rishav-hub) author: John Snow Labs name: xlmroberta_ner_rishav_hub_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `Rishav-hub`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_rishav_hub_base_finetuned_panx_de_4.1.0_3.0_1660430288579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_rishav_hub_base_finetuned_panx_de_4.1.0_3.0_1660430288579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_rishav_hub_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_rishav_hub_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_rishav_hub_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Rishav-hub/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Fast Neural Machine Translation Model from Central Bikol to English author: John Snow Labs name: opus_mt_bcl_en date: 2021-06-01 tags: [open_source, seq2seq, translation, bcl, en, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: bcl target languages: en {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_en_xx_3.1.0_2.4_1622554147726.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_en_xx_3.1.0_2.4_1622554147726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bcl_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Ilocano to English author: John Snow Labs name: opus_mt_ilo_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ilo, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ilo` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ilo_en_xx_2.7.0_2.4_1609165094786.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ilo_en_xx_2.7.0_2.4_1609165094786.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ilo_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_jsl_cased embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm_augmented_cased date: 2021-12-28 tags: [en, clinical, entity_resolution, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbiobert_jsl_cased` Sentence Bert Embeddings. It is trained on the augmented version of the dataset which is used in previous RxNorm resolver models. Additionally, this model returns concept classes of the drugs in all_k_aux_labels column. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_cased_en_3.3.4_2.4_1640687886477.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_cased_en_3.3.4_2.4_1640687886477.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_jsl_cased', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented_cased", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver]) light_model = LightPipeline(rxnorm_pipelineModel) result = light_model.fullAnnotate(["Coumadin 5 mg", "aspirin", "Neurontin 300"]) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_jsl_cased", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented_cased", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val light_model = LightPipeline(rxnorm_pipelineModel) val result = light_model.fullAnnotate(Array("Coumadin 5 mg", "aspirin", "Neurontin 300")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.augmented_cased").predict("""Coumadin 5 mg""") ```

## Results ```bash | | RxNormCode | Resolution | all_k_results | all_k_distances | all_k_cosine_distances | all_k_resolutions | all_k_aux_labels | |---:|-------------:|:-------------------------------------------|:----------------------------------|:----------------------------------|:----------------------------------|:----------------------------------------------------------------|:----------------------------------| | 0 | 855333 | warfarin sodium 5 MG [Coumadin] | 855333:::645146:::432467:::438... | 7.1909:::8.2961:::8.3727:::8.3... | 0.0887:::0.1170:::0.1176:::0.1... | warfarin sodium 5 MG [Coumadin]:::minoxidil 50 MG/ML Topical... | Branded Drug Comp:::Clinical D... | | 1 | 1537020 | aspirin Effervescent Oral Tablet | 1537020:::1191:::437779:::7244... | 0.0000:::0.0000:::8.2570:::8.8... | 0.0000:::0.0000:::0.1147:::0.1... | aspirin Effervescent Oral Tablet:::aspirin:::aspirin / sulfu... | Clinical Drug Form:::Ingredien... | | 2 | 105029 | gabapentin 300 MG Oral Capsule [Neurontin] | 105029:::2180332:::105852:::19... | 8.7466:::10.7744:::11.1256:::1... | 0.1212:::0.1843:::0.1981:::0.2... | gabapentin 300 MG Oral Capsule [Neurontin]:::darolutamide 30... | Branded Drug:::Branded Drug Co... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_augmented_cased| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|972.4 MB| |Case sensitive:|false| --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from mrm8488) author: John Snow Labs name: roberta_qa_ruperta_base_finetuned_squadv2 date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RuPERTa-base-finetuned-squadv2` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ruperta_base_finetuned_squadv2_es_4.2.4_3.0_1669984933151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ruperta_base_finetuned_squadv2_es_4.2.4_3.0_1669984933151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ruperta_base_finetuned_squadv2","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ruperta_base_finetuned_squadv2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ruperta_base_finetuned_squadv2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|470.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-squadv2 --- layout: model title: RCT Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_rct_biobert date: 2022-03-01 tags: [licensed, en, rct, bert, sequence_classification] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT-based](https://github.com/dmis-lab/biobert) classifier that can classify the sections within the abstracts of scientific articles regarding randomized clinical trials (RCT). ## Predicted Entities `BACKGROUND`, `CONCLUSIONS`, `METHODS`, `OBJECTIVE`, `RESULTS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_rct_biobert_en_3.4.1_3.0_1646127001699.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_rct_biobert_en_3.4.1_3.0_1646127001699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ |text |class | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ |[Previous attempts to prevent all the unwanted postoperative responses to major surgery with an epidural hydrophilic opioid , morphine , have not succeeded . The authors ' hypothesis was that the lipophilic opioid fentanyl , infused epidurally close to the spinal-cord opioid receptors corresponding to the dermatome of the surgical incision , gives equal pain relief but attenuates postoperative hormonal and metabolic responses more effectively than does systemic fentanyl .]|[BACKGROUND]| +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_rct_biobert| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://arxiv.org/abs/1710.06071 ## Benchmarking ```bash label precision recall f1-score support BACKGROUND 0.77 0.86 0.81 2000 CONCLUSIONS 0.96 0.95 0.95 2000 METHODS 0.96 0.98 0.97 2000 OBJECTIVE 0.85 0.77 0.81 2000 RESULTS 0.98 0.95 0.96 2000 accuracy 0.9 0.9 0.9 10000 macro-avg 0.9 0.9 0.9 10000 weighted-avg 0.9 0.9 0.9 10000 ``` --- layout: model title: Relation extraction between body parts and direction entities author: John Snow Labs name: re_bodypart_directions date: 2021-01-18 task: Relation Extraction language: en nav_key: models edition: Spark NLP for Healthcare 2.7.1 spark_version: 2.4 tags: [en, relation_extraction, clinical, licensed] supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entites [Internal_organ_or_component, External_body_part_or_region] and Direction entity in clinical texts. `1` : Shows there is a relation between the body part entity and the direction entity, `0` : Shows there is no relation between the body part entity and the direction entity. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb#scrollTo=D8TtVuN-Ee8s){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_en_2.7.1_2.4_1610983817042.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_en_2.7.1_2.4_1610983817042.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `re_bodypart_directions` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABELS | NER MODEL | RE PAIRS | |:----------------------:|:---------------:|:---------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| | re_bodypart_directions | 0,1 | ner_jsl | [“direction-external_body_part_or_region”,
“external_body_part_or_region-direction”,
“direction-internal_organ_or_component”,
“internal_organ_or_component-direction”] |

## Results ```bash | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|-----------------------------|---------------|-------------|------------|-----------------------------|-------------|-------------|---------------|------------| | 0 | 1 | Direction | 35 | 39 | upper | Internal_organ_or_component | 41 | 50 | brain stem | 0.9999989 | | 1 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 59 | 68 | cerebellum | 0.99992585 | | 2 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.9999999 | | 3 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 54 | 57 | left | 0.999811 | | 4 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 75 | 79 | right | 0.9998203 | | 5 | 1 | Direction | 54 | 57 | left | Internal_organ_or_component | 59 | 68 | cerebellum | 1.0 | | 6 | 0 | Direction | 54 | 57 | left | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.97616416 | | 7 | 0 | Internal_organ_or_component | 59 | 68 | cerebellum | Direction | 75 | 79 | right | 0.953046 | | 8 | 1 | Direction | 75 | 79 | right | Internal_organ_or_component | 81 | 93 | basil ganglia | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_bodypart_directions| |Type:|re| |Compatibility:|Spark NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on data gathered and manually annotated by John Snow Labs ## Benchmarking ```bash label recall precision f1 0 0.87 0.9 0.88 1 0.99 0.99 0.99 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from enoriega) author: John Snow Labs name: bert_qa_rule_softmatching date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_softmatching` is a English model originally trained by `enoriega`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_softmatching_en_4.0.0_3.0_1657191287470.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_softmatching_en_4.0.0_3.0_1657191287470.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_softmatching","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_softmatching","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_softmatching| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/enoriega/rule_softmatching --- layout: model title: English RobertaForQuestionAnswering Cased model (from AlirezaBaneshi) author: John Snow Labs name: roberta_qa_autotrain_test2_756523213 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-test2-756523213` is a English model originally trained by `AlirezaBaneshi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523213_en_4.3.0_3.0_1674209108948.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523213_en_4.3.0_3.0_1674209108948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523213","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523213","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_autotrain_test2_756523213| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|415.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AlirezaBaneshi/autotrain-test2-756523213 --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-amharic-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili_sw_4.1.0_3.0_1659353848315.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili_sw_4.1.0_3.0_1659353848315.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_amharic_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-amharic-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Mapping Companies to NASDAQ Stock Screener by Ticker author: John Snow Labs name: finmapper_nasdaq_ticker_stock_screener date: 2023-01-19 tags: [en, finance, licensed, nasdaq, ticker] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model allows you to, given a Ticker, get the following information about a company at Nasdaq Stock Screener: - Country - IPO_Year - Industry - Last_Sale - Market_Cap - Name - Net_Change - Percent_Change - Sector - Ticker - Volume Firstly, you should get the TICKER symbol from the finance text with the `finner_ticker` model, then you can get detailed information about the company with the ChunkMapper model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_ticker_stock_screener_en_1.0.0_3.0_1674157233652.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_ticker_stock_screener_en_1.0.0_3.0_1674157233652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash "Country": "United States", "IPO_Year": "1997", "Industry": "Catalog/Specialty Distribution", "Last_Sale": "$98.12", "Market_Cap": "9.98556270184E11", "Name": "Amazon.com Inc. Common Stock", "Net_Change": "2.85", "Percent_Change": "2.991%", "Sector": "Consumer Discretionary", "Ticker": "AMZN", "Volume": "85412563" ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_nasdaq_ticker_stock_screener| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|584.5 KB| ## References https://www.nasdaq.com/market-activity/stocks/screener --- layout: model title: Korean Bert Embeddings (from kykim) author: John Snow Labs name: bert_embeddings_bert_kor_base date: 2022-04-11 tags: [bert, embeddings, ko, open_source] task: Embeddings language: ko edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-kor-base` is a Korean model orginally trained by `kykim`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_kor_base_ko_3.4.2_3.0_1649675505476.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_kor_base_ko_3.4.2_3.0_1649675505476.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_kor_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ko| |Size:|444.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/kykim/bert-kor-base - https://github.com/kiyoungkim1/LM-kor --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739648723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739648723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_only_classfn_twostage_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0 --- layout: model title: Spanish BERT Sentence Base Uncased Embedding author: John Snow Labs name: sent_bert_base_uncased date: 2021-09-06 tags: [spanish, open_source, bert_sentence_embeddings, uncased, es] task: Embeddings language: es edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BETO is a BERT model trained on a big Spanish corpus. BETO is of size similar to a BERT-Base and was trained with the Whole Word Masking technique. Below you find Tensorflow and Pytorch checkpoints for the uncased and cased versions, as well as some results for Spanish benchmarks comparing BETO with Multilingual BERT as well as other (not BERT-based) models. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_base_uncased_es_3.2.2_3.0_1630926281024.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_base_uncased_es_3.2.2_3.0_1630926281024.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_uncased", "es") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_base_uncased", "es") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu nlu.load("es.embed_sentence.bert.base_uncased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_base_uncased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|es| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/dccuchile/bert-base-spanish-wwm-uncased --- layout: model title: Thai Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_base_thai_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, th, open_source] task: Part of Speech Tagging language: th edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-thai-upos` is a Thai model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_thai_upos_th_3.4.2_3.0_1652092549919.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_thai_upos_th_3.4.2_3.0_1652092549919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_thai_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|th| |Size:|345.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-base-thai-upos - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Detect Genetic Cancer Entities author: John Snow Labs name: ner_cancer_genetics date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN.Pretrained named entity recognition deep learning model for biology and genetics terms. ## Predicted Entities ``DNA``, ``RNA``, ``cell_line``, ``cell_type``, ``protein``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_TUMOR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_en_3.0.0_3.0_1617209717722.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_en_3.0.0_3.0_1617209717722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_cancer_genetics", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_cancer_genetics", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cancer").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""") ```

## Results ```bash +-------------------+---------+ | token|ner_label| +-------------------+---------+ | The| O| | human|B-protein| | KCNJ9|I-protein| | (| O| | Kir|B-protein| | 3.3|I-protein| | ,| O| | GIRK3|B-protein| | )| O| | is| O| | a| O| | member| O| | of| O| | the| O| |G-protein-activated|B-protein| | inwardly|I-protein| | rectifying|I-protein| | potassium|I-protein| | (|I-protein| | GIRK|I-protein| | )|I-protein| | channel|I-protein| | family|I-protein| | .| O| | Here| O| | we| O| | describe| O| | the| O| |genomicorganization| O| | of| O| | the| O| | KCNJ9| B-DNA| | locus| I-DNA| | on| O| | chromosome| B-DNA| | 1q21-23| I-DNA| +-------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cancer_genetics| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on Cancer Genetics (CG) task of the BioNLP Shared Task 2013 with `embeddings_clinical`. https://aclanthology.org/W13-2008/ ## Benchmarking ```bash label tp fp fn prec rec f1 B-cell_line 581 148 151 0.79698217 0.79371583 0.79534566 I-DNA 2751 735 317 0.7891566 0.89667535 0.8394873 I-protein 4416 867 565 0.8358887 0.88656896 0.8604832 B-protein 5288 763 660 0.8739051 0.8890383 0.8814068 I-cell_line 1148 244 301 0.82471263 0.79227054 0.80816615 I-RNA 221 60 27 0.78647685 0.891129 0.83553874 B-RNA 157 40 36 0.79695433 0.8134715 0.8051282 B-cell_type 1127 292 250 0.7942213 0.8184459 0.8061516 I-cell_type 1547 392 263 0.7978339 0.85469615 0.82528675 B-DNA 1513 444 387 0.77312213 0.7963158 0.7845475 Macro-average prec: 0.8069253, rec: 0.84323275, f1: 0.82467955 Micro-average prec: 0.82471186, rec: 0.86377037, f1: 0.84378934 ``` --- layout: model title: Named Entity Recognition Profiling (Biobert) author: John Snow Labs name: ner_profiling_biobert date: 2021-11-03 tags: [ner, ner_profiling, clinical, licensed, en, biobert] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `biobert_pubmed_base_cased`. It has been updated by adding NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `ner_jsl_enriched_biobert`, `ner_clinical_biobert`, `ner_chemprot_biobert`, `ner_jsl_greedy_biobert`, `ner_bionlp_biobert`, `ner_human_phenotype_go_biobert`, `jsl_rd_ner_wip_greedy_biobert`, `ner_posology_large_biobert`, `ner_risk_factors_biobert`, `ner_anatomy_coarse_biobert`, `ner_deid_enriched_biobert`, `ner_human_phenotype_gene_biobert`, `ner_jsl_biobert`, `ner_events_biobert`, `ner_deid_biobert`, `ner_posology_biobert`, `ner_diseases_biobert`, `jsl_ner_wip_greedy_biobert`, `ner_ade_biobert`, `ner_anatomy_biobert`, `ner_cellular_biobert` . {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.2.Pretrained_NER_Profiling_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_3.3.1_2.4_1635977081207.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_3.3.1_2.4_1635977081207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline('ner_profiling_biobert', 'en', 'clinical/models') result = ner_profiling_pipeline.annotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline('ner_profiling_biobert', 'en', 'clinical/models') val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_biobert").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ```

## Results ```bash ******************** ner_diseases_biobert Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('type two diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('HTG-induced pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('poor appetite', 'Disease'), ('vomiting', 'Disease')] ******************** ner_events_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('eight years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('three years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'TEST'), ('BMI', 'TEST'), ('presented', 'OCCURRENCE'), ('a one-week', 'DURATION'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_jsl_biobert Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Obesity'), ('body mass index', 'BMI'), ('BMI ) of 33.5 kg/m2', 'BMI'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index ( BMI )', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_risk_factors_biobert Model Results ******************** [('diabetes mellitus', 'DIABETES'), ('subsequent type two diabetes mellitus', 'DIABETES'), ('obesity', 'OBESE')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_biobert| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel (x21) - NerConverter (x21) - Finisher --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8_en_4.3.0_3.0_1674216234526.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8_en_4.3.0_3.0_1674216234526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_64_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|419.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-8 --- layout: model title: Legal Condemnation Clause Binary Classifier author: John Snow Labs name: legclf_condemnation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `condemnation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `condemnation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_condemnation_clause_en_1.0.0_3.2_1660122247655.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_condemnation_clause_en_1.0.0_3.2_1660122247655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_condemnation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[condemnation]| |[other]| |[other]| |[condemnation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_condemnation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support condemnation 0.96 1.00 0.98 23 other 1.00 0.99 0.99 85 accuracy - - 0.99 108 macro-avg 0.98 0.99 0.99 108 weighted-avg 0.99 0.99 0.99 108 ``` --- layout: model title: Fon asr_fonxlsr TFWav2Vec2ForCTC from chrisjay author: John Snow Labs name: asr_fonxlsr date: 2022-09-24 tags: [wav2vec2, fon, audio, open_source, asr] task: Automatic Speech Recognition language: fon edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_fonxlsr` is a Fon model originally trained by chrisjay. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_fonxlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_fonxlsr_fon_4.2.0_3.0_1664024800003.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_fonxlsr_fon_4.2.0_3.0_1664024800003.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_fonxlsr", "fon")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_fonxlsr", "fon") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_fonxlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fon| |Size:|1.2 GB| --- layout: model title: Translate English to Mon-Khmer languages Pipeline author: John Snow Labs name: translate_en_mkh date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, mkh, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `mkh` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_mkh_xx_2.7.0_2.4_1609689989219.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_mkh_xx_2.7.0_2.4_1609689989219.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_mkh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_mkh", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.mkh').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_mkh| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Clinical Major Concepts to UMLS Code Pipeline author: John Snow Labs name: umls_major_concepts_resolver_pipeline date: 2023-03-30 tags: [en, umls, licensed, pipeline, resolver, clinical] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities (Clinical Major Concepts) with their corresponding UMLS CUI codes. You’ll just feed your text and it will return the corresponding UMLS codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_major_concepts_resolver_pipeline_en_4.3.2_3.2_1680192225130.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_major_concepts_resolver_pipeline_en_4.3.2_3.2_1680192225130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("umls_major_concepts_resolver_pipeline", "en", "clinical/models") pipeline.annotate("The patient complains of pustules after falling from stairs. She has been advised Arthroscopy by her primary care pyhsician") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= PretrainedPipeline("umls_major_concepts_resolver_pipeline", "en", "clinical/models") val pipeline.annotate("The patient complains of pustules after falling from stairs. She has been advised Arthroscopy by her primary care pyhsician") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.umls_major_concepts_resolver").predict("""The patient complains of pustules after falling from stairs. She has been advised Arthroscopy by her primary care pyhsician""") ```

## Results ```bash +-----------+-----------------------------------+---------+ |chunk |ner_label |umls_code| +-----------+-----------------------------------+---------+ |pustules |Sign_or_Symptom |C0241157 | |stairs |Daily_or_Recreational_Activity |C4300351 | |Arthroscopy|Therapeutic_or_Preventive_Procedure|C0179144 | +-----------+-----------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_major_concepts_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.0 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: English DistilBertForQuestionAnswering model (from machine2049) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_squad_ date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad_distilbert` is a English model originally trained by `machine2049`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad__en_4.0.0_3.0_1654726959258.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad__en_4.0.0_3.0_1654726959258.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_machine2049").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_squad_| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/machine2049/distilbert-base-uncased-finetuned-squad_distilbert --- layout: model title: Swedish asr_test_by_marma TFWav2Vec2ForCTC from marma author: John Snow Labs name: asr_test_by_marma date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_test_by_marma` is a Swedish model originally trained by marma. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_test_by_marma_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_test_by_marma_sv_4.2.0_3.0_1664116267395.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_test_by_marma_sv_4.2.0_3.0_1664116267395.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_test_by_marma", "sv")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_test_by_marma", "sv") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_test_by_marma| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|sv| |Size:|756.2 MB| --- layout: model title: Chinese Word Segmentation author: John Snow Labs name: wordseg_msra date: 2021-01-03 task: Word Segmentation language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, word_segmentation, zh, cn] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know _a priori_ which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. References: - Xue, Nianwen. "Chinese word segmentation as character tagging." International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_msra_zh_2.7.0_2.4_1609693916888.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_msra_zh_2.7.0_2.4_1609693916888.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline as a substitute of the Tokenizer stage.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.pretrained('wordseg_msra', 'zh')\ .setInputCols("document")\ .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler, word_segmenter]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['然而，这样的处理也衍生了一些问题。']], ["text"]) result = model.transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_msra", "zh") .setInputCols("document") .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("然而，这样的处理也衍生了一些问题。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，这样的处理也衍生了一些问题。"""] token_df = nlu.load('zh.segment_words.msra').predict(text, output_level='token') token_df ```

## Results ```bash +----------------------------------+--------------------------------------------------------+ |text |result | +----------------------------------+--------------------------------------------------------+ |然而，这样的处理也衍生了一些问题。|[然而, ，, 这样, 的, 处理, 也, 衍生, 了, 一些, 问题, 。]| +----------------------------------+--------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_msra| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|zh| ## Data Source We trained this model on the Microsoft Research Asia (MSRA) data set available on the Second International Chinese Word Segmentation Bakeoff [SIGHAN 2005](http://sighan.cs.uchicago.edu/bakeoff2005) ## Benchmarking ```bash | Model | precision | recall | f1-score | |---------------|--------------|--------------|--------------| | WORSEG_CTB | 0,6453 | 0,6341 | 0,6397 | | WORDSEG_WEIBO | 0,5454 | 0,5655 | 0,5553 | | WORDSEG_MSRA | 0,5984 | 0,6088 | 0,6035 | | WORDSEG_PKU | 0,6094 | 0,6321 | 0,6206 | | WORDSEG_LARGE | 0,6326 | 0,6269 | 0,6297 | ``` --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from V3RX2000) author: John Snow Labs name: xlmroberta_ner_v3rx2000_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `V3RX2000`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_v3rx2000_base_finetuned_panx_all_xx_4.1.0_3.0_1660427771019.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_v3rx2000_base_finetuned_panx_all_xx_4.1.0_3.0_1660427771019.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_v3rx2000_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_v3rx2000_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_v3rx2000_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/V3RX2000/xlm-roberta-base-finetuned-panx-all --- layout: model title: English Part of Speech Tagger (Large, UPOS-Universal Part-Of-Speech) author: John Snow Labs name: roberta_pos_roberta_large_english_upos date: 2022-05-03 tags: [roberta, pos, part_of_speech, en, open_source] task: Part of Speech Tagging language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-large-english-upos` is a English model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_large_english_upos_en_3.4.2_3.0_1651596140502.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_large_english_upos_en_3.4.2_3.0_1651596140502.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_roberta_large_english_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/roberta-large-english-upos - https://universaldependencies.org/en/ - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Stop Words Cleaner for Polish author: John Snow Labs name: stopwords_pl date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: pl edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, pl] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_pl_pl_2.5.4_2.4_1594742438519.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_pl_pl_2.5.4_2.4_1594742438519.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_pl", "pl") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_pl", "pl") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej."""] stopword_df = nlu.load('pl.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=5, result='Oprócz', metadata={'sentence': '0'}), Row(annotatorType='token', begin=7, end=11, result='bycia', metadata={'sentence': '0'}), Row(annotatorType='token', begin=13, end=18, result='królem', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=26, result='północy', metadata={'sentence': '0'}), Row(annotatorType='token', begin=27, end=27, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_pl| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|pl| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Duplets) author: John Snow Labs name: distilbert_qa_duplets_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Duplets`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_duplets_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768419922.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_duplets_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768419922.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_duplets_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_duplets_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_duplets_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Duplets/distilbert-base-uncased-finetuned-squad --- layout: model title: French CamemBert Embeddings (from Henrywang) author: John Snow Labs name: camembert_embeddings_Henrywang_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Henrywang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Henrywang_generic_model_fr_3.4.4_3.0_1653986322898.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Henrywang_generic_model_fr_3.4.4_3.0_1653986322898.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Henrywang_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Henrywang/dummy-model --- layout: model title: Modern Greek (1453-) asr_greek_lsr_1 TFWav2Vec2ForCTC from skylord author: John Snow Labs name: pipeline_asr_greek_lsr_1 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_greek_lsr_1` is a Modern Greek (1453-) model originally trained by skylord. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_greek_lsr_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_greek_lsr_1_el_4.2.0_3.0_1664111543823.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_greek_lsr_1_el_4.2.0_3.0_1664111543823.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_greek_lsr_1', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_greek_lsr_1", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_greek_lsr_1| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Ganda asr_wav2vec2_luganda_by_cahya TFWav2Vec2ForCTC from cahya author: John Snow Labs name: asr_wav2vec2_luganda_by_cahya date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_luganda_by_cahya` is a Ganda model originally trained by cahya. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_luganda_by_cahya_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_cahya_lg_4.2.0_3.0_1664037739573.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_cahya_lg_4.2.0_3.0_1664037739573.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_luganda_by_cahya", "lg")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_luganda_by_cahya", "lg") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_luganda_by_cahya| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lg| |Size:|1.2 GB| --- layout: model title: Fast Neural Machine Translation Model from San Salvador Kongo to English author: John Snow Labs name: opus_mt_kwy_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, kwy, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `kwy` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_kwy_en_xx_2.7.0_2.4_1609170973770.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_kwy_en_xx_2.7.0_2.4_1609170973770.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_kwy_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: BioBERT Embeddings (PMC) author: John Snow Labs name: biobert_pmc_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pmc_base_cased_en_2.6.0_2.4_1598343018425.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pmc_base_cased_en_2.6.0_2.4_1598343018425.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("biobert_pmc_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("biobert_pmc_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.pmc_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_biobert_pmc_base_cased_embeddings I [0.0654267892241478, 0.06330983340740204, 0.13... hate [0.3058323264122009, 0.4778319299221039, -0.09... cancer [0.3130614757537842, 0.024675076827406883, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728_de_4.2.0_3.0_1664115088643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728_de_4.2.0_3.0_1664115088643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_10_austria_0_s728| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Abkhazian asr_wav2vec2_common_voice_ab_demo TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: asr_wav2vec2_common_voice_ab_demo date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_common_voice_ab_demo` is a Abkhazian model originally trained by patrickvonplaten. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_common_voice_ab_demo_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_common_voice_ab_demo_ab_4.2.0_3.0_1664042256123.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_common_voice_ab_demo_ab_4.2.0_3.0_1664042256123.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_common_voice_ab_demo", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_common_voice_ab_demo", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_common_voice_ab_demo| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|1.2 GB| --- layout: model title: Mapping Entities with Corresponding RxNorm Codes and Normalized Names author: John Snow Labs name: rxnorm_normalized_mapper date: 2022-09-29 tags: [en, clinical, licensed, rxnorm, chunk_mapping] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities with their corresponding RxNorm codes and normalized RxNorm resolutions. ## Predicted Entities `rxnorm_code`, `normalized_name` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_normalized_mapper_en_4.1.0_3.0_1664443862683.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_normalized_mapper_en_4.1.0_3.0_1664443862683.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+-----------+--------------------------------------------------------------+ |ner_chunk |rxnorm_code|normalized_name | +------------------------------+-----------+--------------------------------------------------------------+ |Zyrtec 10 MG |1011483 |cetirizine hydrochloride 10 MG [Zyrtec] | |Adapin 10 MG Oral Capsule |1000050 |doxepin hydrochloride 10 MG Oral Capsule [Adapin] | |Septi-Soothe 0.5 Topical Spray|1000046 |chlorhexidine diacetate 0.5 MG/ML Topical Spray [Septi-Soothe]| +------------------------------+-----------+--------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_normalized_mapper| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|10.7 MB| --- layout: model title: English DistilBertForQuestionAnswering Cased model (from Slavka) author: John Snow Labs name: distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-finetuned-log-parser-winlogbeat` is a English model originally trained by `Slavka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat_en_4.0.0_3.0_1654723355284.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat_en_4.0.0_3.0_1654723355284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.base_cased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_bert_base_cased_finetuned_log_parser_winlogbeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Slavka/bert-base-cased-finetuned-log-parser-winlogbeat --- layout: model title: Summarize clinical notes author: John Snow Labs name: summarizer_clinical_jsl date: 2023-03-25 tags: [en, licensed, clinical, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Summarize clinical notes, encounters, critical care notes, discharge notes, reports, etc. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/MEDICAL_TEXT_SUMMARIZATION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/32.Medical_Text_Summarization.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_en_4.3.1_3.0_1679772340755.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_en_4.3.1_3.0_1679772340755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler().setInputCol('text').setOutputCol('document') summarizer = MedicalSummarizer.pretrained("summarizer_clinical_jsl", "en", "clinical/models").setInputCols(['document'])\ .setOutputCol('summary')\ .setMaxTextLength(512)\ .setMaxNewTokens(512) pipeline = sparknlp.base.Pipeline(stages=[ document, summarizer ]) text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val summarizer = MedicalSummarizer.pretrained("summarizer_clinical_jsl", "en", "clinical/models") .setInputCols(['document']) .setOutputCol('summary') .setMaxTextLength(512) .setMaxNewTokens(512) val pipeline = new Pipeline().setStages(Array(document_assembler, summarizer)) val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" val data = Seq(text).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash A 78-year-old female with hypertension, syncope, and spinal stenosis returns for recheck. She denies chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. She is on multiple medications and has Elocon cream and Synalar cream for rash. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_jsl| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.1 MB| ## References Trained on in-house curated dataset --- layout: model title: Sentence Entity Resolver for billable ICD10-CM HCC codes (Slim, JSL Medium Bert) author: John Snow Labs name: sbertresolve_icd10cm_slim_billable_hcc_med date: 2021-05-21 tags: [licensed, clinical, en, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using sentence embeddings. This model has been augmented with synonyms, and synonyms having low cosine similarity are dropped, making the model slim. It utilises fine-tuned `sbert_jsl_medium_uncased` Sentence Bert Model. ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for aux_label parameter in the metadata to get HCC status. The HCC status can be divided to get further information: billable status, hcc status, and hcc score. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_slim_billable_hcc_med_en_3.0.4_2.4_1621590174924.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_slim_billable_hcc_med_en_3.0.4_2.4_1621590174924.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler().setInputCol("text").setOutputCol("document") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_uncased","en","clinical/models")\ .setInputCols(["document"])\ .setOutputCol("sbert_embeddings") icd10_resolver = SentenceEntityResolverModel\ .pretrained("sbertresolve_icd10cm_slim_billable_hcc_med","en", "clinical/models")\ .setInputCols(["document", "sbert_embeddings"])\ .setOutputCol("icd10cm_code")\ .setDistanceFunction("EUCLIDEAN")\ .setReturnCosineDistances(True) bert_pipeline_icd = Pipeline(stages = [document_assembler, sbert_embedder, icd10_resolver]) data = spark.createDataFrame([["metastatic lung cancer"]]).toDF("text") results = bert_pipeline_icd.fit(data).transform(data) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased","en","clinical/models") .setInputCols(Array("document")) .setOutputCol("sbert_embeddings") val icd10_resolver = SentenceEntityResolverModel .pretrained("sbertresolve_icd10cm_slim_billable_hcc_med","en", "clinical/models") .setInputCols(Array("document", "sbert_embeddings")) .setOutputCol("icd10cm_code") .setDistanceFunction("EUCLIDEAN") .setReturnCosineDistances(True) val bert_pipeline_icd = new Pipeline().setStages(Array(document_assembler, sbert_embedder, icd10_resolver)) val data = Seq("metastatic lung cancer").toDF("text") val result = bert_pipeline_icd.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd10cm.slim_billable_hcc_med").predict("""metastatic lung cancer""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_icd10cm_slim_billable_hcc_med| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[icd10_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: English DistilBertForQuestionAnswering model (from twmkn9) author: John Snow Labs name: distilbert_qa_base_uncased_squad2 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-squad2` is a English model originally trained by `twmkn9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_en_4.0.0_3.0_1654727261779.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_en_4.0.0_3.0_1654727261779.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.distil_bert.base_uncased.by_twmkn9").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/twmkn9/distilbert-base-uncased-squad2 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_128 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_128_zh_4.2.4_3.0_1670325968748.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_128_zh_4.2.4_3.0_1670325968748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_128| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|15.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-128 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Legal Transition Services Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_transition_services_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, transition_services, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_transition_services_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `transition-services-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `transition-services-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_transition_services_agreement_bert_en_1.0.0_3.0_1669372317483.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_transition_services_agreement_bert_en_1.0.0_3.0_1669372317483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_transition_services_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[transition-services-agreement]| |[other]| |[other]| |[transition-services-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_transition_services_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.98 0.98 82 transition-services-agreement 0.94 0.97 0.96 34 accuracy - - 0.97 116 macro-avg 0.97 0.97 0.97 116 weighted-avg 0.97 0.97 0.97 116 ``` --- layout: model title: Legal Letter Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_letter_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, letter, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_letter_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `letter-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `letter-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_letter_agreement_bert_en_1.0.0_3.0_1669315660461.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_letter_agreement_bert_en_1.0.0_3.0_1669315660461.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_letter_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[letter-agreement]| |[other]| |[other]| |[letter-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_letter_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support letter-agreement 0.94 0.87 0.90 38 other 0.93 0.97 0.95 65 accuracy - - 0.93 103 macro-avg 0.93 0.92 0.93 103 weighted-avg 0.93 0.93 0.93 103 ``` --- layout: model title: Stop Words Cleaner for Galician author: John Snow Labs name: stopwords_gl date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: gl edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, gl] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_gl_gl_2.5.4_2.4_1594742441210.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_gl_gl_2.5.4_2.4_1594742441210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_gl", "gl") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Ademais de ser o rei do norte, John Snow é un médico inglés e un líder no desenvolvemento da anestesia e a hixiene médica.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_gl", "gl") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Ademais de ser o rei do norte, John Snow é un médico inglés e un líder no desenvolvemento da anestesia e a hixiene médica.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Ademais de ser o rei do norte, John Snow é un médico inglés e un líder no desenvolvemento da anestesia e a hixiene médica."""] stopword_df = nlu.load('gl.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=6, result='Ademais', metadata={'sentence': '0'}), Row(annotatorType='token', begin=17, end=19, result='rei', metadata={'sentence': '0'}), Row(annotatorType='token', begin=24, end=28, result='norte', metadata={'sentence': '0'}), Row(annotatorType='token', begin=29, end=29, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=31, end=34, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_gl| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|gl| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Turkish BertForQuestionAnswering model (from yunusemreemik) author: John Snow Labs name: bert_qa_logo_qna_model date: 2022-06-02 tags: [tr, open_source, question_answering, bert] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `logo-qna-model` is a Turkish model orginally trained by `yunusemreemik`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_logo_qna_model_tr_4.0.0_3.0_1654188164539.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_logo_qna_model_tr_4.0.0_3.0_1654188164539.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_logo_qna_model","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_logo_qna_model","tr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.bert.by_yunusemreemik").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_logo_qna_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tr| |Size:|412.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/yunusemreemik/logo-qna-model --- layout: model title: Swedish asr_Wav2Vec2_large_xlsr_welsh TFWav2Vec2ForCTC from Srulikbdd author: John Snow Labs name: asr_Wav2Vec2_large_xlsr_welsh date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_large_xlsr_welsh` is a Swedish model originally trained by Srulikbdd. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Wav2Vec2_large_xlsr_welsh_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_large_xlsr_welsh_sv_4.2.0_3.0_1664115056212.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_large_xlsr_welsh_sv_4.2.0_3.0_1664115056212.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Wav2Vec2_large_xlsr_welsh", "sv")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Wav2Vec2_large_xlsr_welsh", "sv") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Wav2Vec2_large_xlsr_welsh| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|sv| |Size:|1.2 GB| --- layout: model title: Bert for Sequence Classification (Question vs Statement) author: John Snow Labs name: bert_sequence_classifier_question_statement date: 2021-11-04 tags: [question, statement, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.3.2 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Trained to add sentence classifying capabilities to distinguish between Question vs Statements. This model was imported from Hugging Face (https://huggingface.co/shahrukhx01/question-vs-statement-classifier), and trained based on Haystack (https://github.com/deepset-ai/haystack/issues/611). ## Predicted Entities `question`, `statement` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_question_statement_en_3.3.2_3.0_1636038134936.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_question_statement_en_3.3.2_3.0_1636038134936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") seq = BertForSequenceClassification.pretrained('bert_sequence_classifier_question_statement', 'en')\ .setInputCols(["token", "sentence"])\ .setOutputCol("label")\ .setCaseSensitive(True) pipeline = Pipeline(stages = [ documentAssembler, sentenceDetector, tokenizer, seq]) test_sentences = ["""What feature in your car did you not realize you had until someone else told you about it? Years ago, my Dad bought me a cute little VW Beetle. The first day I had it, me and my BFF were sitting in my car looking at everything. When we opened the center console, we had quite the scare. Inside was a hollowed out, plastic phallic looking object with tiny spikes on it. My friend and I literally screamed in horror. It was clear to us that somehow someone left their “toy” in my new car! We were shook, as they say. This was my car, I had to do something. So, I used a pen to pick up the nasty looking thing and threw it out. We freaked out about how gross it was and then we forgot about it… until my Dad called me. My Dad said: How’s the new car? Have you seen the flower holder in the center console? To summarize, we thought a flower vase was an XXX item… In our defense, this is a picture of a VW Beetle flower holder."""] import pandas as pd data=spark.createDataFrame(pd.DataFrame({'text': test_sentences})) res = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val seq = BertForSequenceClassification.pretrained("bert_sequence_classifier_question_statement", "en") .setInputCols(Array("token", "sentence")) .setOutputCol("label") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, seq)) val test_sentences = "What feature in your car did you not realize you had until someone else told you about it? Years ago, my Dad bought me a cute little VW Beetle. The first day I had it, me and my BFF were sitting in my car looking at everything. When we opened the center console, we had quite the scare. Inside was a hollowed out, plastic phallic looking object with tiny spikes on it. My friend and I literally screamed in horror. It was clear to us that somehow someone left their “toy” in my new car! We were shook, as they say. This was my car, I had to do something. So, I used a pen to pick up the nasty looking thing and threw it out. We freaked out about how gross it was and then we forgot about it… until my Dad called me. My Dad said: How’s the new car? Have you seen the flower holder in the center console? To summarize, we thought a flower vase was an XXX item… In our defense, this is a picture of a VW Beetle flower holder." val example = Seq(test_sentences).toDF("text") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.question_vs_statement").predict("""What feature in your car did you not realize you had until someone else told you about it? Years ago, my Dad bought me a cute little VW Beetle. The first day I had it, me and my BFF were sitting in my car looking at everything. When we opened the center console, we had quite the scare. Inside was a hollowed out, plastic phallic looking object with tiny spikes on it. My friend and I literally screamed in horror. It was clear to us that somehow someone left their “toy” in my new car! We were shook, as they say. This was my car, I had to do something. So, I used a pen to pick up the nasty looking thing and threw it out. We freaked out about how gross it was and then we forgot about it… until my Dad called me. My Dad said: How’s the new car? Have you seen the flower holder in the center console? To summarize, we thought a flower vase was an XXX item… In our defense, this is a picture of a VW Beetle flower holder.""") ```

## Results ```bash +------------------------------------------------------------------------------------------+---------+ | sentence| label| +------------------------------------------------------------------------------------------+---------+ |What feature in your car did you not realize you had until someone else told you about it?| question| | Years ago, my Dad bought me a cute little VW Beetle.|statement| | The first day I had it, me and my BFF were sitting in my car looking at everything.|statement| | When we opened the center console, we had quite the scare.|statement| | Inside was a hollowed out, plastic phallic looking object with tiny spikes on it.|statement| | My friend and I literally screamed in horror.|statement| | It was clear to us that somehow someone left their “toy” in my new car!|statement| | We were shook, as they say.|statement| | This was my car, I had to do something.|statement| | So, I used a pen to pick up the nasty looking thing and threw it out.|statement| |We freaked out about how gross it was and then we forgot about it… until my Dad called me.|statement| | My Dad said: How’s the new car?| question| | Have you seen the flower holder in the center console?| question| | To summarize, we thought a flower vase was an XXX item…|statement| | In our defense, this is a picture of a VW Beetle flower holder.|statement| +------------------------------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_question_statement| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[label]| |Language:|en| |Case sensitive:|true| ## Data Source https://github.com/deepset-ai/haystack/issues/611 ## Benchmarking ```bash Extracted from https://github.com/deepset-ai/haystack/issues/611 precision recall f1-score support statement 0.94 0.94 0.94 16105 question 0.96 0.96 0.96 26198 accuracy 0.95 42303 macro avg 0.95 0.95 0.95 42303 weighted avg 0.95 0.95 0.95 42303 ``` --- layout: model title: Legal Court Judgment Prediction (Portuguese) author: John Snow Labs name: legclf_judgment_prediction date: 2023-04-06 tags: [pt, licensed, legal, classification, tensorflow] task: Text Classification language: pt edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model which identifies the court decisions in the State Supreme Court, including the following classes; - no: The appeal was denied - partial: For partially favourable decisions - yes: For fully favourable decisions ## Predicted Entities `no`, `partial`, `yes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_judgment_prediction_pt_1.0.0_3.0_1680778981035.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_judgment_prediction_pt_1.0.0_3.0_1680778981035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler= nlp.DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = nlp.Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") seq_classifier = legal.BertForSequenceClassification.pretrained("legclf_judgment_prediction", "pt", "legal/models") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = nlp.Pipeline(stages=[ document_assembler, tokenizer , seq_classifier ]) # simple examples example = spark.createDataFrame([["PENAL. PROCESSO PENAL. APELAÇÃO. HOMICÍDIO QUALIFICADO. ARGUIÇÃO DE NULIDADE EM DECORRÊNCIA DA INSCONSTITUCIONALIDADE DO ARTIGO 457 DO CÓDIGO DE PROCESSO PENAL. AFASTADA. PLEITO DE REDIMENSIONAMENTO DA PENA. DOSIMETRIA QUE MERECE RETOQUES. AFASTADA A VALORAÇÃO DESFAVORÁVEL DAS CIRCUNSTÂNCIAS JUDICIAIS DOS ANTECEDENTES E DA PERSONALIDADE DO AGENTE. MANTIDA A CULPABILIDADE, CIRCUNSTÂNCIAS DO DELITO E CONSEQUÊNCIAS DO CRIME. APELO CONHECIDO E PARCIALMENTE PROVIDO. 1 Não há falar em ocorrência de nulidade não caso concreto, não existindo qualquer inconstitucionalidade em virtude do texto legal do ARTIGO 457 do Código de Processo Penal, não tendo ocorrido o adiamento da sessão do júri em virtude da ausência do acusado, conforme alegado. Pelo contrário, este foi devidamente intimado por edital e, mesmo assim, restou ausente. 2 A justificativa apresentada pelo magistrado singular acerca da culpabilidade considerou a alta reprovabilidade da conduta do réu, em virtude da premeditação e frieza na prática delitiva, considerando que o acusado foi até a casa da vítima com o intuito de ceifar a sua vida, experimentando assim a consequência da transgressão, estando acertada a valoração negativa desta circunstância judicial."]]).toDF("text") result = pipeline.fit(example).transform(example) # result is a DataFrame result.select("text", "class.result").show(truncate=100) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+---------+ | text| result| +----------------------------------------------------------------------------------------------------+---------+ |PENAL. PROCESSO PENAL. APELAÇÃO. HOMICÍDIO QUALIFICADO. ARGUIÇÃO DE NULIDADE EM DECORRÊNCIA DA IN...|[partial]| +----------------------------------------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_judgment_prediction| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|pt| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## Benchmarking ```bash label precision recall f1-score support no 0.76 0.77 0.76 86 partial 0.79 0.71 0.75 75 yes 0.71 0.78 0.74 76 accuracy - - 0.75 237 macro-avg 0.75 0.75 0.75 237 weighted-avg 0.75 0.75 0.75 237 ``` --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-128-finetuned-squad-seed-42` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42_en_4.0.0_3.0_1657184284783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42_en_4.0.0_3.0_1657184284783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_42| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-128-finetuned-squad-seed-42 --- layout: model title: English image_classifier_vit__spectrogram ViTForImageClassification from prashanth0205 author: John Snow Labs name: image_classifier_vit__spectrogram date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit__spectrogram` is a English model originally trained by prashanth0205. ## Predicted Entities `female`, `male` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit__spectrogram_en_4.1.0_3.0_1660170988681.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit__spectrogram_en_4.1.0_3.0_1660170988681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit__spectrogram", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit__spectrogram", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit__spectrogram| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from tli8hf) author: John Snow Labs name: bert_qa_unqover_bert_base_uncased_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unqover-bert-base-uncased-squad` is a English model orginally trained by `tli8hf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_unqover_bert_base_uncased_squad_en_4.0.0_3.0_1654192543690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_unqover_bert_base_uncased_squad_en_4.0.0_3.0_1654192543690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_unqover_bert_base_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_unqover_bert_base_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_tli8hf").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_unqover_bert_base_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tli8hf/unqover-bert-base-uncased-squad --- layout: model title: Pipeline to Detect Units and Measurements author: John Snow Labs name: ner_measurements_clinical_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, measurements, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_measurements_clinical](https://nlp.johnsnowlabs.com/2021/04/01/ner_measurements_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_pipeline_en_3.4.1_3.0_1647870532389.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_measurements_clinical_pipeline_en_3.4.1_3.0_1647870532389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_measurements_clinical_pipeline", "en", "clinical/models") pipeline.annotate("EXAMPLE MEDICAL TEXT") ``` ```scala val pipeline = new PretrainedPipeline("ner_measurements_clinical_pipeline", "en", "clinical/models") pipeline.annotate("EXAMPLE MEDICAL TEXT") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical_measurements.pipeline").predict("""EXAMPLE MEDICAL TEXT""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_measurements_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English asr_wav2vec2_cetuc_sid_voxforge_mls_0 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: asr_wav2vec2_cetuc_sid_voxforge_mls_0 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_cetuc_sid_voxforge_mls_0` is a English model originally trained by joaoalvarenga. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_cetuc_sid_voxforge_mls_0_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_cetuc_sid_voxforge_mls_0_en_4.2.0_3.0_1664022744764.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_cetuc_sid_voxforge_mls_0_en_4.2.0_3.0_1664022744764.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_cetuc_sid_voxforge_mls_0", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_cetuc_sid_voxforge_mls_0", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_cetuc_sid_voxforge_mls_0| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: SDOH Insurance Type For Classification author: John Snow Labs name: genericclassifier_sdoh_insurance_type_sbiobert_cased_mli date: 2023-04-28 tags: [en, insurance, sdoh, social_determinants, public_health, classificaiton, licensed] task: Text Classification language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting insurance type. In this classifier, we know/assume that the patient **has insurance**. If the patient's insurance type is not mentioned or not known, it is regarded as an "Other" type of insurance. And if the patient's insurance is one of "Tricare" or "VA (Veterans Affair)", it is labeled as Military. The model is trained by using GenericClassifierApproach annotator. `Employer`: Employer insurance. `Medicaid`: Medicaid insurance. `Medicare`: Medicare insurance. `Military`: "Tricare" or "VA (Veterans Affair)" insurance. `Private`: Private insurance. `Other`: Other insurance or insurance type is not mentioned in the clinical notes or is not known. ## Predicted Entities `Employer`, `Medicaid`, `Medicare`, `Military`, `Private`, `Other` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/social_determinant){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_insurance_type_sbiobert_cased_mli_en_4.4.0_3.0_1682694596560.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_insurance_type_sbiobert_cased_mli_en_4.4.0_3.0_1682694596560.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", 'en','clinical/models')\ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") features_asm = FeaturesAssembler()\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("features") generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_insurance_type_sbiobert_cased_mli", 'en', 'clinical/models')\ .setInputCols(["features"])\ .setOutputCol("prediction") pipeline = Pipeline(stages=[ document_assembler, sentence_embeddings, features_asm, generic_classifier ]) text_list = [ "The patient has VA insurance.", "She is under Medicare insurance", "The patient has good coverage of Private insurance", """Medical File for John Smith, Male, Age 42 Chief Complaint: Patient complains of nausea, vomiting, and shortness of breath. History of Present Illness: The patient has a history of hypertension and diabetes, which are both poorly controlled. The patient has been feeling unwell for the past week, with symptoms including nausea, vomiting, and shortness of breath. Upon examination, the patient was found to have a high serum creatinine level of 5.8 mg/dL, indicating renal failure. Past Medical History: The patient has a history of hypertension and diabetes, which have been poorly controlled due to poor medication adherence. The patient also has a history of smoking, which has been a contributing factor to the development of renal failure. Medications: The patient is currently taking Metformin and Lisinopril for the management of diabetes and hypertension, respectively. However, due to poor Medicaid coverage, the patient is unable to afford some of the medications prescribed by his physician. Insurance Status: The patient has Medicaid insurance, which provides poor coverage for some of the medications needed to manage his medical conditions, including those related to his renal failure. Physical Examination: Upon physical examination, the patient appears pale and lethargic. Blood pressure is 160/100 mmHg, heart rate is 90 beats per minute, and respiratory rate is 20 breaths per minute. There is diffuse abdominal tenderness on palpation, and lung auscultation reveals bilateral rales. Diagnosis: The patient is diagnosed with acute renal failure, likely due to uncontrolled hypertension and poorly managed diabetes. Treatment: The patient is started on intravenous fluids and insulin to manage his blood sugar levels. Due to the patient's poor Medicaid coverage, the physician works with the patient to identify alternative medications that are more affordable and will still provide effective management of his medical conditions. Follow-Up: The patient is advised to follow up with his primary care physician for ongoing management of his renal failure and other medical conditions. The patient is also referred to a nephrologist for further evaluation and management of his renal failure. """, """Certainly, here is an example case study for a patient with private insurance: Case Study for Emily Chen, Female, Age 38 Chief Complaint: Patient reports chronic joint pain and stiffness. History of Present Illness: The patient has been experiencing chronic joint pain and stiffness, particularly in the hands, knees, and ankles. The pain is worse in the morning and improves throughout the day. The patient has also noticed some swelling and redness in the affected joints. Past Medical History: The patient has a history of osteoarthritis, which has been gradually worsening over the past several years. The patient has tried over-the-counter pain relievers and joint supplements, but has not found significant relief. Medications: The patient is currently taking over-the-counter pain relievers and joint supplements for the management of her osteoarthritis. Insurance Status: The patient has private insurance, which provides comprehensive coverage for her medical care, including specialist visits and prescription medications. Physical Examination: Upon physical examination, the patient has tenderness and swelling in multiple joints, particularly the hands, knees, and ankles. Range of motion is limited due to pain and stiffness. Diagnosis: The patient is diagnosed with osteoarthritis, a chronic degenerative joint disease that causes pain, swelling, and stiffness in the affected joints. Treatment: The patient is prescribed a nonsteroidal anti-inflammatory drug (NSAID) to manage pain and inflammation. The physician also recommends physical therapy to improve range of motion and strengthen the affected joints. The patient is advised to continue taking joint supplements for ongoing joint health. Follow-Up: The patient is advised to follow up with the physician in 4-6 weeks to monitor response to treatment and make any necessary adjustments. The patient is also referred to a rheumatologist for further evaluation and management of her osteoarthritis.""", """ Medical File for John Doe, Male, Age 72 Chief Complaint: Patient reports shortness of breath and fatigue. History of Present Illness: The patient has been experiencing shortness of breath and fatigue for the past several weeks. The patient reports difficulty performing daily activities and has noticed a decrease in exercise tolerance. Past Medical History: The patient has a history of hypertension, hyperlipidemia, and coronary artery disease. The patient has undergone a coronary artery bypass graft (CABG) surgery in the past. Medications: The patient is currently taking several medications, including a beta blocker, a statin, and a diuretic, for the management of his medical conditions. Insurance Status: The patient has good coverage of Medicare insurance, which provides comprehensive coverage for his medical care, including specialist visits, diagnostic tests, and prescription medications. Physical Examination: Upon physical examination, the patient has crackles in the lungs and peripheral edema. Blood pressure is elevated, and heart sounds are irregular. Diagnosis: The patient is diagnosed with congestive heart failure, a chronic condition in which the heart cannot pump blood effectively to meet the body's needs. Treatment: The patient is admitted to the hospital for further evaluation and management of his congestive heart failure. Treatment includes diuresis to remove excess fluid, medication management to control blood pressure and heart rate, and oxygen therapy to improve breathing. The patient is also advised to follow a low-sodium diet and to monitor his fluid intake closely. Follow-Up: The patient is advised to follow up with his primary care physician and cardiologist regularly to monitor his heart function and adjust treatment as necessary. The patient is also referred to cardiac rehabilitation to improve his exercise tolerance and overall cardiovascular health."""] df = spark.createDataFrame(text_list, StringType()).toDF("text") result = pipeline.fit(df).transform(df) result.select("text", "prediction.result").show(truncate=100) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols("document") .setOutputCol("sentence_embeddings") val features_asm = new FeaturesAssembler() .setInputCols("sentence_embeddings") .setOutputCol("features") val generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_insurance_type_sbiobert_cased_mli", "en", "clinical/models") .setInputCols("features") .setOutputCol("prediction") val pipeline = new PipelineModel().setStages(Array( document_assembler, sentence_embeddings, features_asm, generic_classifier)) val data = Seq(Array( "The patient has VA insurance.", "She is under Medicare insurance", "The patient has good coverage of Private insurance", """Medical File for John Smith, Male, Age 42 Chief Complaint: Patient complains of nausea, vomiting, and shortness of breath. History of Present Illness: The patient has a history of hypertension and diabetes, which are both poorly controlled. The patient has been feeling unwell for the past week, with symptoms including nausea, vomiting, and shortness of breath. Upon examination, the patient was found to have a high serum creatinine level of 5.8 mg/dL, indicating renal failure. Past Medical History: The patient has a history of hypertension and diabetes, which have been poorly controlled due to poor medication adherence. The patient also has a history of smoking, which has been a contributing factor to the development of renal failure. Medications: The patient is currently taking Metformin and Lisinopril for the management of diabetes and hypertension, respectively. However, due to poor Medicaid coverage, the patient is unable to afford some of the medications prescribed by his physician. Insurance Status: The patient has Medicaid insurance, which provides poor coverage for some of the medications needed to manage his medical conditions, including those related to his renal failure. Physical Examination: Upon physical examination, the patient appears pale and lethargic. Blood pressure is 160/100 mmHg, heart rate is 90 beats per minute, and respiratory rate is 20 breaths per minute. There is diffuse abdominal tenderness on palpation, and lung auscultation reveals bilateral rales. Diagnosis: The patient is diagnosed with acute renal failure, likely due to uncontrolled hypertension and poorly managed diabetes. Treatment: The patient is started on intravenous fluids and insulin to manage his blood sugar levels. Due to the patient's poor Medicaid coverage, the physician works with the patient to identify alternative medications that are more affordable and will still provide effective management of his medical conditions. Follow-Up: The patient is advised to follow up with his primary care physician for ongoing management of his renal failure and other medical conditions. The patient is also referred to a nephrologist for further evaluation and management of his renal failure. """, """Certainly, here is an example case study for a patient with private insurance: Case Study for Emily Chen, Female, Age 38 Chief Complaint: Patient reports chronic joint pain and stiffness. History of Present Illness: The patient has been experiencing chronic joint pain and stiffness, particularly in the hands, knees, and ankles. The pain is worse in the morning and improves throughout the day. The patient has also noticed some swelling and redness in the affected joints. Past Medical History: The patient has a history of osteoarthritis, which has been gradually worsening over the past several years. The patient has tried over-the-counter pain relievers and joint supplements, but has not found significant relief. Medications: The patient is currently taking over-the-counter pain relievers and joint supplements for the management of her osteoarthritis. Insurance Status: The patient has private insurance, which provides comprehensive coverage for her medical care, including specialist visits and prescription medications. Physical Examination: Upon physical examination, the patient has tenderness and swelling in multiple joints, particularly the hands, knees, and ankles. Range of motion is limited due to pain and stiffness. Diagnosis: The patient is diagnosed with osteoarthritis, a chronic degenerative joint disease that causes pain, swelling, and stiffness in the affected joints. Treatment: The patient is prescribed a nonsteroidal anti-inflammatory drug (NSAID) to manage pain and inflammation. The physician also recommends physical therapy to improve range of motion and strengthen the affected joints. The patient is advised to continue taking joint supplements for ongoing joint health. Follow-Up: The patient is advised to follow up with the physician in 4-6 weeks to monitor response to treatment and make any necessary adjustments. The patient is also referred to a rheumatologist for further evaluation and management of her osteoarthritis.""", """ Medical File for John Doe, Male, Age 72 Chief Complaint: Patient reports shortness of breath and fatigue. History of Present Illness: The patient has been experiencing shortness of breath and fatigue for the past several weeks. The patient reports difficulty performing daily activities and has noticed a decrease in exercise tolerance. Past Medical History: The patient has a history of hypertension, hyperlipidemia, and coronary artery disease. The patient has undergone a coronary artery bypass graft (CABG) surgery in the past. Medications: The patient is currently taking several medications, including a beta blocker, a statin, and a diuretic, for the management of his medical conditions. Insurance Status: The patient has good coverage of Medicare insurance, which provides comprehensive coverage for his medical care, including specialist visits, diagnostic tests, and prescription medications. Physical Examination: Upon physical examination, the patient has crackles in the lungs and peripheral edema. Blood pressure is elevated, and heart sounds are irregular. Diagnosis: The patient is diagnosed with congestive heart failure, a chronic condition in which the heart cannot pump blood effectively to meet the body's needs. Treatment: The patient is admitted to the hospital for further evaluation and management of his congestive heart failure. Treatment includes diuresis to remove excess fluid, medication management to control blood pressure and heart rate, and oxygen therapy to improve breathing. The patient is also advised to follow a low-sodium diet and to monitor his fluid intake closely. Follow-Up: The patient is advised to follow up with his primary care physician and cardiologist regularly to monitor his heart function and adjust treatment as necessary. The patient is also referred to cardiac rehabilitation to improve his exercise tolerance and overall cardiovascular health.""")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+----------+ | text| result| +----------------------------------------------------------------------------------------------------+----------+ | The patient has VA insurance.|[Military]| | She is under Medicare insurance|[Medicare]| |Medical File for John Smith, Male, Age 42\n\nChief Complaint: Patient complains of nausea, vomiti...|[Medicaid]| |Certainly, here is an example case study for a patient with private insurance:\n\nCase Study for ...| [Private]| |\nMedical File for John Doe, Male, Age 72\n\nChief Complaint: Patient reports shortness of breath...|[Medicare]| +----------------------------------------------------------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_insurance_type_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH project ## Benchmarking ```bash label precision recall f1-score support Employer 0.67 0.82 0.74 17 Medicaid 0.89 0.80 0.84 61 Medicare 0.85 0.89 0.87 38 Military 0.76 0.89 0.82 18 Other 0.56 0.45 0.50 11 Private 0.80 0.77 0.79 31 accuracy - - 0.81 176 macro-avg 0.75 0.77 0.76 176 weighted-avg 0.81 0.81 0.81 176 ``` --- layout: model title: Translate Wallisian to English Pipeline author: John Snow Labs name: translate_wls_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, wls, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `wls` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_wls_en_xx_2.7.0_2.4_1609688000528.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_wls_en_xx_2.7.0_2.4_1609688000528.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_wls_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_wls_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.wls.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_wls_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: BERT Embeddings (Base Cased) author: John Snow Labs name: bert_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus. The details are described in the paper "[BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_en_2.6.0_2.4_1598340336670.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_en_2.6.0_2.4_1598340336670.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_base_cased_embeddings I [0.43879568576812744, -0.40160006284713745, 0.... love [0.21737590432167053, -0.3865768313407898, -0.... NLP [-0.16226479411125183, -0.053727392107248306, ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/bert_cased_L-12_H-768_A-12/1](https://tfhub.dev/google/bert_cased_L-12_H-768_A-12/1) --- layout: model title: Multilingual BERT Embeddings (Base Cased) author: John Snow Labs name: bert_multi_cased date: 2020-08-25 task: Embeddings language: xx edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, xx] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus. The details are described in the paper "[BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_multi_cased_xx_2.6.0_2.4_1598341875191.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_multi_cased_xx_2.6.0_2.4_1598341875191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_multi_cased", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love Spark NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_multi_cased", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love Spark NLP"] embeddings_df = nlu.load('xx.embed.bert_multi_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash xx_embed_bert_multi_cased_embeddings token [0.31631314754486084, -0.5579454898834229, 0.1... I [-0.1488783359527588, -0.27264419198036194, -0... love [0.0496230386197567, -0.43625175952911377, -0.... Spark [-0.2838578224182129, -0.7103433012962341, 0.4... NLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_multi_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[xx]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3 --- layout: model title: Korean BertForQuestionAnswering model (from bespin-global) author: John Snow Labs name: bert_qa_klue_bert_base_aihub_mrc date: 2022-06-02 tags: [ko, open_source, question_answering, bert] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `klue-bert-base-aihub-mrc` is a Korean model orginally trained by `bespin-global`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_klue_bert_base_aihub_mrc_ko_4.0.0_3.0_1654188035750.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_klue_bert_base_aihub_mrc_ko_4.0.0_3.0_1654188035750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_klue_bert_base_aihub_mrc","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_klue_bert_base_aihub_mrc","ko") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.klue.bert.base_aihub.by_bespin-global").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_klue_bert_base_aihub_mrc| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ko| |Size:|413.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bespin-global/klue-bert-base-aihub-mrc - https://github.com/KLUE-benchmark/KLUE - https://www.bespinglobal.com/ - https://aihub.or.kr/aidata/86 --- layout: model title: Relation Extraction between Tests, Results, and Dates author: John Snow Labs name: re_test_result_date date: 2021-02-24 tags: [licensed, en, clinical, relation_extraction] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.4 spark_version: 2.4 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between lab test names, their findings, measurements, results, and date. ## Predicted Entities `is_finding_of`, `is_result_of`, `is_date_of`, `O`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CLINICAL_DATE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb#scrollTo=D8TtVuN-Ee8s){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_test_result_date_en_2.7.4_2.4_1614168615976.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_test_result_date_en_2.7.4_2.4_1614168615976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, PerceptronModel, DependencyParserModel, WordEmbeddingsModel, NerDLModel, NerConverter, RelationExtractionModel In the table below, `re_test_result_date` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:-------------------:|:------------------------------------------------------:|:---------:|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | re_test_result_date | is_finding_of,
is_result_of,
is_date_of,
O | ner_jsl | [“test-test_result”,
“test_result-test”,
“test-date”, “date-test”,
“test-imagingfindings”,
“imagingfindings-test”,
“test-ekg_findings”,
“ekg_findings-test”,
“date-test_result”,
“test_result-date”,
“date-imagingfindings”,
“imagingfindings-date”,
“date-ekg_findings”,
“ekg_findings-date”] |

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("embeddings") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") ner_tagger = MedicalNerModel().pretrained('jsl_ner_wip_clinical',"en","clinical/models")\ .setInputCols("sentences", "tokens", "embeddings")\ .setOutputCol("ner_tags") ner_chunker = NerConverterInternal()\ .setInputCols(["sentences", "tokens", "ner_tags"])\ .setOutputCol("ner_chunks") dependency_parser = DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") re_model = RelationExtractionModel().pretrained("re_test_result_date", "en", 'clinical/models')\ .setInputCols(["embeddings", "pos_tags", "ner_chunks", "dependencies"])\ .setOutputCol("relations")\ .setMaxSyntacticDistance(4)\ .setPredictionThreshold(0.9)\ .setRelationPairs(["external_body_part_or_region-test"])# Possible relation pairs. Default: All Relations. nlp_pipeline = Pipeline(stages=[document_assembler, sentencer, tokenizer, word_embeddings, pos_tagger, ner_tagger, ner_chunker, dependency_parser, re_model]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("""He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%""") ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentencer = new SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val pos_tagger = PerceptronModel().pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val ner_tagger = MedicalNerModel().pretrained("jsl_ner_wip_clinical","en","clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_chunker = new NerConverterInternal() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val dependency_parser = DependencyParserModel().pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") val re_model = RelationExtractionModel().pretrained("re_test_result_date", "en", "clinical/models") .setInputCols(Array("embeddings", "pos_tags", "ner_chunks", "dependencies")) .setOutputCol("relations") .setMaxSyntacticDistance(4) .setPredictionThreshold(0.9) .setRelationPairs("external_body_part_or_region-test")# Possible relation pairs. Default: All Relations. val nlp_pipeline = new Pipeline().setStages(Array(document_assembler, sentencer, tokenizer, word_embeddings, pos_tagger, ner_tagger, ner_chunker, dependency_parser, re_model)) val text = """He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%""" val data = Seq(text).toDS.toDF("text") val results = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.test_result_date").predict("""He was advised chest X-ray or CT scan after checking his SpO2 which was <= 93%""") ```

## Results ```bash | index | relations | entity1 | chunk1 | entity2 | chunk2 | |-------|--------------|--------------|---------------------|--------------|---------| | 0 | O | TEST | chest X-ray | MEASUREMENTS | 93% | | 1 | O | TEST | CT scan | MEASUREMENTS | 93% | | 2 | is_result_of | TEST | SpO2 | MEASUREMENTS | 93% | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_test_result_date| |Type:|re| |Compatibility:|Healthcare NLP 2.7.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| ## Data Source Trained on internal data. ## Benchmarking ```bash | relation | prec | |-----------------|------| | O | 0.77 | | is_finding_of | 0.80 | | is_result_of | 0.96 | | is_date_of | 0.94 | ``` --- layout: model title: English image_classifier_vit_base_mri ViTForImageClassification from raedinkhaled author: John Snow Labs name: image_classifier_vit_base_mri date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_mri` is a English model originally trained by raedinkhaled. ## Predicted Entities `cad`, `healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_mri_en_4.1.0_3.0_1660168752129.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_mri_en_4.1.0_3.0_1660168752129.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_mri", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_mri", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_mri| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForQuestionAnswering model (from sunitha) author: John Snow Labs name: distilbert_qa_base_uncased_3feb_2022_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-3feb-2022-finetuned-squad` is a English model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_3feb_2022_finetuned_squad_en_4.0.0_3.0_1654723730880.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_3feb_2022_finetuned_squad_en_4.0.0_3.0_1654723730880.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_3feb_2022_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_3feb_2022_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_sunitha").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_3feb_2022_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/distilbert-base-uncased-3feb-2022-finetuned-squad --- layout: model title: Medical Spell Checker Pipeline author: John Snow Labs name: spellcheck_clinical_pipeline date: 2022-04-14 tags: [spellcheck, medical, medical_spell_checker, spell_corrector, spell_pipeline, en, licensed, clinical] task: Spell Check language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained medical spellchecker pipeline is built on the top of [spellcheck_clinical](https://nlp.johnsnowlabs.com/2022/04/14/spellcheck_clinical_en_2_4.html) model. This pipeline is for PySpark 2.4.x users with SparkNLP 3.4.1. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_pipeline_en_3.4.1_2.4_1649930943224.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_pipeline_en_3.4.1_2.4_1649930943224.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("spellcheck_clinical_pipeline", "en", "clinical/models") example = ["Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.", "With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.", "Abdomen is sort, nontender, and nonintended.", "Patient not showing pain or any wealth problems.", "No cute distress"] pipeline.fullAnnotate(example) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("spellcheck_clinical_pipeline", "en", "clinical/models") val example = Array("Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.", "With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.", "Abdomen is sort, nontender, and nonintended.", "Patient not showing pain or any wealth problems.", "No cute distress") pipeline.fullAnnotate(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.spell.clinical.pipeline").predict("""Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.""") ```

## Results ```bash [{'checked': ['With','the','cell','of','physical','therapy','the','patient','was','ambulated','and','on','postoperative',',','the','patient','tolerating','a','post','surgical','soft','diet','.'], 'document': ['Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.'], 'token': ['Witth','the','hell','of','phisical','terapy','the','patient','was','imbulated','and','on','postoperative',',','the','impatient','tolerating','a','post','curgical','soft','diet','.']}, {'checked': ['With','pain','well','controlled','on','oral','pain','medications',',','she','was','discharged','to','rehabilitation','facility','.'], 'document': ['With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.'], 'token': ['With','paint','wel','controlled','on','orall','pain','medications',',','she','was','discharged','too','reihabilitation','facilitay','.']}, {'checked': ['Abdomen','is','soft',',','nontender',',','and','nondistended','.'], 'document': ['Abdomen is sort, nontender, and nonintended.'], 'token': ['Abdomen','is','sort',',','nontender',',','and','nonintended','.']}, {'checked': ['Patient','not','showing','pain','or','any','health','problems','.'], 'document': ['Patient not showing pain or any wealth problems.'], 'token': ['Patient','not','showing','pain','or','any','wealth','problems','.']}, {'checked': ['No', 'acute', 'distress'], 'document': ['No cute distress'], 'token': ['No', 'cute', 'distress']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|141.3 MB| ## Included Models - DocumentAssembler - TokenizerModel - ContextSpellCheckerModel --- layout: model title: German asr_exp_w2v2t_vp_s946 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2t_vp_s946 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_vp_s946` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2t_vp_s946_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s946_de_4.2.0_3.0_1664110443438.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s946_de_4.2.0_3.0_1664110443438.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2t_vp_s946', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2t_vp_s946", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2t_vp_s946| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Living Species author: John Snow Labs name: bert_token_classifier_ner_living_species date: 2022-06-27 tags: [es, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: es edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Spanish which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. It is trained on the [LivingNER](https://temu.bsc.es/livingner/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_es_3.5.3_3.0_1656316616890.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_es_3.5.3_3.0_1656316616890.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "es", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, ner_model, ner_converter ]) data = spark.createDataFrame([["""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "es", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("ner") .setCaseSensitive(True) .setMaxSentenceLength(512) val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, ner_model, ner_converter)) val data = Seq("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.classify.bert_token.ner_living_species").predict("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""") ```

## Results ```bash +--------------+-------+ |ner_chunk |label | +--------------+-------+ |Lactante varón|HUMAN | |familiares |HUMAN | |personales |HUMAN | |neonatal |HUMAN | |legumbres |SPECIES| |lentejas |SPECIES| |garbanzos |SPECIES| |legumbres |SPECIES| |madre |HUMAN | |Cacahuete |SPECIES| |padres |HUMAN | +--------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://temu.bsc.es/livingner/](https://temu.bsc.es/livingner/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.96 0.99 0.97 3281 B-SPECIES 0.89 0.94 0.91 3712 I-HUMAN 0.86 0.75 0.80 297 I-SPECIES 0.88 0.90 0.89 1732 micro-avg 0.91 0.94 0.93 9022 macro-avg 0.90 0.89 0.89 9022 weighted-avg 0.91 0.94 0.93 9022 ``` --- layout: model title: Legal Management Clause Binary Classifier author: John Snow Labs name: legclf_management_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `management` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `management` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_management_clause_en_1.0.0_3.2_1660123720911.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_management_clause_en_1.0.0_3.2_1660123720911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_management_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[management]| |[other]| |[other]| |[management]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_management_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support management 0.96 0.90 0.93 71 other 0.95 0.98 0.96 136 accuracy - - 0.95 207 macro-avg 0.95 0.94 0.95 207 weighted-avg 0.95 0.95 0.95 207 ``` --- layout: model title: Relation Extraction between Tumors and Sizes author: John Snow Labs name: re_oncology_size_wip date: 2022-09-26 tags: [licensed, clinical, oncology, relation_extraction, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links Tumor_Size extractions to their corresponding Tumor_Finding extractions. ## Predicted Entities `is_size_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_oncology_size_wip_en_4.0.0_3.0_1664230171831.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_oncology_size_wip_en_4.0.0_3.0_1664230171831.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk1 | entity1 | chunk2 | entity2 | relation | confidence | |:---------|:--------------|:---------|:--------------|:-----------|-------------:| | 2 cm | Tumor_Size | mass | Tumor_Finding | is_size_of | 0.853271 | | tumor | Tumor_Finding | 3 cm | Tumor_Size | is_size_of | 0.815623 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_oncology_size_wip| |Type:|re| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Size:|267.5 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 is_size_of 0.89 0.77 0.83 O 0.75 0.88 0.81 macro-avg 0.82 0.82 0.82 ``` --- layout: model title: English image_classifier_vit_exper6_mesum5 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_exper6_mesum5 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_exper6_mesum5` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper6_mesum5_en_4.1.0_3.0_1660168138363.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper6_mesum5_en_4.1.0_3.0_1660168138363.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_exper6_mesum5", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_exper6_mesum5", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_exper6_mesum5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: English Named Entity Recognition (from CouchCat) author: John Snow Labs name: distilbert_ner_ma_ner_v7_distil date: 2022-05-16 tags: [distilbert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ma_ner_v7_distil` is a English model orginally trained by `CouchCat`. ## Predicted Entities `MATR`, `PERS`, `TIME`, `MISC`, `PAD`, `PROD`, `BRND` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_ma_ner_v7_distil_en_3.4.2_3.0_1652721967576.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_ma_ner_v7_distil_en_3.4.2_3.0_1652721967576.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_ma_ner_v7_distil| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CouchCat/ma_ner_v7_distil --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_4_H_256_A_4_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-4_H-256_A-4_squad2` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_256_A_4_squad2_en_4.0.0_3.0_1654185268072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_256_A_4_squad2_en_4.0.0_3.0_1654185268072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_256_A_4_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_256_A_4_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.uncased_4l_256d_a4a_256d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_256_A_4_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|42.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-256_A-4_squad2 --- layout: model title: English BertForQuestionAnswering model (from AnonymousSub) author: John Snow Labs name: bert_qa_bert_FT_newsqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_FT_newsqa` is a English model orginally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_FT_newsqa_en_4.0.0_3.0_1654185068956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_FT_newsqa_en_4.0.0_3.0_1654185068956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.bert.ft.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/bert_FT_newsqa --- layout: model title: English image_classifier_vit_new_york_tokyo_london ViTForImageClassification from Suzana author: John Snow Labs name: image_classifier_vit_new_york_tokyo_london date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_new_york_tokyo_london` is a English model originally trained by Suzana. ## Predicted Entities `London`, `New York`, `Tokyo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_new_york_tokyo_london_en_4.1.0_3.0_1660171162315.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_new_york_tokyo_london_en_4.1.0_3.0_1660171162315.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_new_york_tokyo_london", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_new_york_tokyo_london", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_new_york_tokyo_london| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: French Legal Roberta Embeddings author: John Snow Labs name: roberta_base_french_legal date: 2023-02-16 tags: [fr, french, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: fr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-french-roberta-base` is a French model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_french_legal_fr_4.2.4_3.0_1676580048854.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_french_legal_fr_4.2.4_3.0_1676580048854.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_french_legal", "fr")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_french_legal", "fr") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_french_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|416.2 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-french-roberta-base --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from tiennvcs) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_infov date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-infovqa` is a English model originally trained by `tiennvcs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_infov_en_4.3.0_3.0_1672768056130.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_infov_en_4.3.0_3.0_1672768056130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_infov","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_infov","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_infov| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tiennvcs/distilbert-base-uncased-finetuned-infovqa --- layout: model title: English ElectraForQuestionAnswering model (from ptran74) Version-2 author: John Snow Labs name: electra_qa_DSPFirst_Finetuning_2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `DSPFirst-Finetuning-2` is a English model originally trained by `ptran74`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_2_en_4.0.0_3.0_1655919435626.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_2_en_4.0.0_3.0_1655919435626.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.electra.finetuning_2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_DSPFirst_Finetuning_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ptran74/DSPFirst-Finetuning-2 - https://github.gatech.edu/pages/VIP-ITS/textbook_SQuAD_explore/explore/textbookv1.0/textbook/ --- layout: model title: Оcr small for handwritten text author: John Snow Labs name: ocr_small_handwritten date: 2022-02-17 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 3.3.3 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Ocr small model for recognise handwritten text based on TrOcr architecture. The TrOCR model was proposed in TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. TrOCR consists of an image Transformer encoder and an autoregressive text Transformer decoder to perform optical character recognition (OCR). The abstract from the paper is the following: Text recognition is a long-standing research problem for document digitalization. Existing approaches for text recognition are usually built based on CNN for image understanding and RNN for char-level text generation. In addition, another language model is usually needed to improve the overall accuracy as a post-processing step. In this paper, we propose an end-to-end text recognition approach with pre-trained image Transformer and text Transformer models, namely TrOCR, which leverages the Transformer architecture for both image understanding and wordpiece-level text generation. The TrOCR model is simple but effective, and can be pre-trained with large-scale synthetic data and fine-tuned with human-labeled datasets. Experiments show that the TrOCR model outperforms the current state-of-the-art models on both printed and handwritten text recognition tasks. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_handwritten_en_3.3.3_2.4_1645080334390.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_handwritten_en_3.3.3_2.4_1645080334390.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ocr = ImageToTextv2().pretrained("ocr_small_handwritten", "en", "clinical/ocr") ocr.setInputCols(["image"]) ocr.setOutputCol("text") result = ocr.transform(image_text_lines_df).collect() print(result[0].text)

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ocr = ImageToTextv2().pretrained("ocr_small_handwritten", "en", "clinical/ocr") ocr.setInputCols(["image"]) ocr.setOutputCol("text") result = ocr.transform(image_text_lines_df).collect() print(result[0].text) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ocr_small_handwritten| |Type:|ocr| |Compatibility:|Visual NLP 3.3.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|146.7 MB| --- layout: model title: Fast Neural Machine Translation Model from Afro-Asiatic languages to English author: John Snow Labs name: opus_mt_afa_en date: 2021-06-01 tags: [open_source, seq2seq, translation, afa, en, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: afa target languages: en {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_afa_en_xx_3.1.0_2.4_1622562470297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_afa_en_xx_3.1.0_2.4_1622562470297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_afa_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for ICD-10-CM (general 3 character codes - augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_generalised_augmented date: 2023-05-31 tags: [licensed, en, clinical, entity_resolution, icd10cm] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD-10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It predicts ICD-10-CM codes up to 3 characters (according to ICD-10-CM code structure the first three characters represent general type of the injury or disease). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_generalised_augmented_en_4.4.2_3.0_1685508789416.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_generalised_augmented_en_4.4.2_3.0_1685508789416.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+----------+---------------------------------------------------------------------------+-----------------------------------------------------------------+ | ner_chunk| entity|icd10_code| resolutions| all_codes| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+-----------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24|[gestational diabetes mellitus [gestational diabetes mellitus], history ...| [O24, Z86, Z87]| |subsequent type two diabetes mellitus|PROBLEM| O24|[pre-existing type 2 diabetes mellitus [pre-existing type 2 diabetes mel...| [O24, E11, E13, Z86]| | obesity|PROBLEM| E66|[obesity [obesity, unspecified], obese [body mass index [bmi] 40.0-44.9,...| [E66, Z68, Q13, Z86, E34, H35, Z83, Q55]| | a body mass index|PROBLEM| Z68|[finding of body mass index [body mass index [bmi] 40.0-44.9, adult], ob...| [Z68, E66, R22, R41, M62, P29, R19, R89, M21]| | polyuria|PROBLEM| R35|[polyuria [polyuria], polyuric state (disorder) [diabetes insipidus], he...|[R35, E23, R31, R82, N40, E72, O04, R30, R80, E88, N03, P96, N02]| | polydipsia|PROBLEM| R63|[polydipsia [polydipsia], psychogenic polydipsia [other impulse disorder...|[R63, F63, E23, O40, G47, M79, R06, H53, I44, Q30, I45, R00, M35]| | poor appetite|PROBLEM| R63|[poor appetite [anorexia], poor feeding [feeding problem of newborn, uns...|[R63, P92, R43, E86, R19, F52, Z72, R06, Z76, R53, R45, F50, R10]| | vomiting|PROBLEM| R11|[vomiting [vomiting], periodic vomiting [cyclical vomiting, in migraine,...| [R11, G43, P92]| | a respiratory tract infection|PROBLEM| J98|[respiratory tract infection [other specified respiratory disorders], up...| [J98, J06, A49, J22, J20, Z59, T17, J04, Z13, J18, P28, J39]| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+-----------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_generalised_augmented| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|1.4 GB| |Case sensitive:|false| --- layout: model title: English asr_wav2vec2_base_cynthia_tedlium_2500_v2 TFWav2Vec2ForCTC from huyue012 author: John Snow Labs name: pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_cynthia_tedlium_2500_v2` is a English model originally trained by huyue012. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2_en_4.2.0_3.0_1664040558353.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2_en_4.2.0_3.0_1664040558353.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_cynthia_tedlium_2500_v2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.2 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Portuguese asr_bp_voxforge1_xlsr TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_bp_voxforge1_xlsr date: 2022-09-26 tags: [wav2vec2, pt, audio, open_source, asr] task: Automatic Speech Recognition language: pt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bp_voxforge1_xlsr` is a Portuguese model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bp_voxforge1_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bp_voxforge1_xlsr_pt_4.2.0_3.0_1664193020078.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bp_voxforge1_xlsr_pt_4.2.0_3.0_1664193020078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bp_voxforge1_xlsr", "pt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bp_voxforge1_xlsr", "pt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bp_voxforge1_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|pt| |Size:|1.2 GB| --- layout: model title: RxNorm Cd ChunkResolver author: John Snow Labs name: chunkresolve_rxnorm_cd_clinical date: 2021-04-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities RxNorm Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_cd_clinical_en_3.0.0_3.0_1618603400196.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_cd_clinical_en_3.0.0_3.0_1618603400196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... rxnorm_resolver = ChunkEntityResolverModel()\ .pretrained('chunkresolve_rxnorm_cd_clinical', 'en', "clinical/models")\ .setEnableLevenshtein(True)\ .setNeighbours(200).setAlternatives(5).setDistanceWeights([3,11,0,0,0,9])\ .setInputCols(['token', 'chunk_embeddings'])\ .setOutputCol('rxnorm_resolution')\ .setPoolingStrategy("MAX") pipeline_rxnorm = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, rxnorm_resolver]) model = pipeline_rxnorm.fit(spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val rxnorm_resolver = ChunkEntityResolverModel() .pretrained('chunkresolve_rxnorm_cd_clinical', 'en', "clinical/models") .setEnableLevenshtein(True) .setNeighbours(200).setAlternatives(5).setDistanceWeights(Array(3,11,0,0,0,9)) .setInputCols('token', 'chunk_embeddings') .setOutputCol('rxnorm_resolution') .setPoolingStrategy("MAX") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, rxnorm_resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG. She had been on dapagliflozin for six months at the time of presentation.").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | chunk| entity| target_text| code|confidence| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ | metformin|TREATMENT|metFORMIN compounding powder:::Metformin Hydrochloride Powder:::metFORMIN 500 mg oral tablet:::me...| 601021| 0.2364| | glipizide|TREATMENT|Glipizide Powder:::Glipizide Crystal:::Glipizide Tablets:::glipiZIDE 5 mg oral tablet:::glipiZIDE...| 241604| 0.3647| |dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG|TREATMENT|Ezetimibe and Atorvastatin Tablets:::Amlodipine and Atorvastatin Tablets:::Atorvastatin Calcium T...|1422084| 0.3407| | dapagliflozin|TREATMENT|Dapagliflozin Tablets:::dapagliflozin 5 mg oral tablet:::dapagliflozin 10 mg oral tablet:::Dapagl...|1488568| 0.7070| +---------------------------------------------------------------+---------+----------------------------------------------------------------------------------------------------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_rxnorm_cd_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[rxnorm]| |Language:|en| --- layout: model title: English BertForQuestionAnswering model (from ruselkomp) author: John Snow Labs name: bert_qa_tests_finetuned_squad_test_bert_2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tests-finetuned-squad-test-bert-2` is a English model orginally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_tests_finetuned_squad_test_bert_2_en_4.0.0_3.0_1654192311570.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_tests_finetuned_squad_test_bert_2_en_4.0.0_3.0_1654192311570.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tests_finetuned_squad_test_bert_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_tests_finetuned_squad_test_bert_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.v2.by_ruselkomp").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tests_finetuned_squad_test_bert_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/tests-finetuned-squad-test-bert-2 --- layout: model title: English image_classifier_vit_generation_xyz ViTForImageClassification from chradden author: John Snow Labs name: image_classifier_vit_generation_xyz date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_generation_xyz` is a English model originally trained by chradden. ## Predicted Entities `Generation Alpha`, `Millennials`, `Generation X`, `Generation Z`, `Baby Boomers` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_generation_xyz_en_4.1.0_3.0_1660171707791.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_generation_xyz_en_4.1.0_3.0_1660171707791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_generation_xyz", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_generation_xyz", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_generation_xyz| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForTokenClassification Cased model (from Neurona) author: John Snow Labs name: distilbert_token_classifier_cpener_test date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cpener-test` is a English model originally trained by `Neurona`. ## Predicted Entities `cpe_version`, `cpe_product`, `cpe_vendor` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_cpener_test_en_4.3.0_3.0_1677881384855.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_cpener_test_en_4.3.0_3.0_1677881384855.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_cpener_test| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Neurona/cpener-test --- layout: model title: Fast Neural Machine Translation Model from Multiple languages to English author: John Snow Labs name: opus_mt_mul_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, mul, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `mul` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_mul_en_xx_2.7.0_2.4_1609166361160.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_mul_en_xx_2.7.0_2.4_1609166361160.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_mul_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Problems, Tests and Treatments (ner_clinical_large) author: John Snow Labs name: ner_clinical_large_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_clinical_large](https://nlp.johnsnowlabs.com/2021/03/31/ner_clinical_large_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_large_pipeline_en_4.3.0_3.2_1678876271920.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_large_pipeline_en_4.3.0_3.2_1678876271920.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_clinical_large_pipeline", "en", "clinical/models") text = '''The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_clinical_large_pipeline", "en", "clinical/models") val text = "The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical_large.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------------------------------------|--------:|------:|:------------|-------------:| | 0 | the G-protein-activated inwardly rectifying potassium (GIRK | 48 | 106 | TREATMENT | 0.6926 | | 1 | the genomicorganization | 142 | 164 | TREATMENT | 0.80715 | | 2 | a candidate gene forType II diabetes mellitus | 210 | 254 | PROBLEM | 0.754343 | | 3 | byapproximately | 380 | 394 | TREATMENT | 0.7924 | | 4 | single nucleotide polymorphisms | 464 | 494 | TREATMENT | 0.636967 | | 5 | aVal366Ala substitution | 532 | 554 | PROBLEM | 0.53615 | | 6 | an 8 base-pair | 561 | 574 | PROBLEM | 0.607733 | | 7 | insertion/deletion | 581 | 598 | PROBLEM | 0.8692 | | 8 | Ourexpression studies | 601 | 621 | TEST | 0.89975 | | 9 | the transcript in various humantissues | 648 | 685 | PROBLEM | 0.83306 | | 10 | fat andskeletal muscle | 749 | 770 | PROBLEM | 0.778133 | | 11 | furtherstudies | 830 | 843 | PROBLEM | 0.8789 | | 12 | the KCNJ9 protein | 864 | 880 | TREATMENT | 0.561033 | | 13 | evaluation | 892 | 901 | TEST | 0.9981 | | 14 | Type II diabetes | 940 | 955 | PROBLEM | 0.698967 | | 15 | the treatment | 1025 | 1037 | TREATMENT | 0.81195 | | 16 | breast cancer | 1042 | 1054 | PROBLEM | 0.9604 | | 17 | the standard therapy | 1067 | 1086 | TREATMENT | 0.757767 | | 18 | anthracyclines | 1125 | 1138 | TREATMENT | 0.9999 | | 19 | taxanes | 1144 | 1150 | TREATMENT | 0.9999 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becasv2_2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becasv2-2` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_2_en_4.3.0_3.0_1672767690511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_2_en_4.3.0_3.0_1672767690511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becasv2_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becasv2-2 --- layout: model title: English asr_autonlp_hindi_asr TFWav2Vec2ForCTC from abhishek author: John Snow Labs name: asr_autonlp_hindi_asr date: 2022-09-26 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_autonlp_hindi_asr` is a English model originally trained by abhishek. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_autonlp_hindi_asr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_autonlp_hindi_asr_en_4.2.0_3.0_1664195323489.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_autonlp_hindi_asr_en_4.2.0_3.0_1664195323489.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_autonlp_hindi_asr", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_autonlp_hindi_asr", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_autonlp_hindi_asr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English RobertaForSequenceClassification Cased model (from jawadhussein462) author: John Snow Labs name: roberta_classifier_autotrain_neurips_chanllenge_1287149282 date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification, tensorflow] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true recommended: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-neurips_chanllenge-1287149282` is a English model originally trained by `jawadhussein462`. ## Predicted Entities `1`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_neurips_chanllenge_1287149282_en_4.2.4_3.0_1670624021899.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_neurips_chanllenge_1287149282_en_4.2.4_3.0_1670624021899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_neurips_chanllenge_1287149282","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_neurips_chanllenge_1287149282","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_autotrain_neurips_chanllenge_1287149282| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/jawadhussein462/autotrain-neurips_chanllenge-1287149282 --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_EManuals_RoBERTa_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `EManuals_RoBERTa_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_EManuals_RoBERTa_squad2.0_en_4.0.0_3.0_1655726695679.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_EManuals_RoBERTa_squad2.0_en_4.0.0_3.0_1655726695679.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_EManuals_RoBERTa_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_EManuals_RoBERTa_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.emanuals.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_EManuals_RoBERTa_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/EManuals_RoBERTa_squad2.0 --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_facebook TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_german_by_facebook date: 2022-09-24 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_facebook` is a German model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_german_by_facebook_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_facebook_de_4.2.0_3.0_1664026408621.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_facebook_de_4.2.0_3.0_1664026408621.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_german_by_facebook", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_german_by_facebook", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_german_by_facebook| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|332.6 MB| --- layout: model title: Detect Radiology Concepts - WIP (biobert) author: John Snow Labs name: jsl_rd_ner_wip_greedy_biobert date: 2021-07-26 tags: [licensed, clinical, en, ner] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.1.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract clinical entities from Radiology reports using pretrained NER model. ## Predicted Entities `Test_Result`, `OtherFindings`, `BodyPart`, `ImagingFindings`, `Disease_Syndrome_Disorder`, `ImagingTest`, `Measurements`, `Procedure`, `Score`, `Test`, `Medical_Device`, `Direction`, `Symptom`, `Imaging_Technique`, `ManualFix`, `Units` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_biobert_en_3.1.3_3.0_1627305153541.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_biobert_en_3.1.3_3.0_1627305153541.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained('biobert_pubmed_base_cased') \ .setInputCols(['sentence', 'token']) \ .setOutputCol('embeddings') clinical_ner = MedicalNerModel.pretrained("jsl_rd_ner_wip_greedy_biobert", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma."]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols("sentence", "token") .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("jsl_rd_ner_wip_greedy_biobert", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.radiology.wip_greedy_biobert").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | chunk | entity | |---:|:----------------------|:--------------------------| | 0 | Bilateral | Direction | | 1 | breast | BodyPart | | 2 | ultrasound | ImagingTest | | 3 | ovoid mass | ImagingFindings | | 4 | 0.5 x 0.5 x 0.4 | Measurements | | 5 | cm | Units | | 6 | left | Direction | | 7 | shoulder | BodyPart | | 8 | mass | ImagingFindings | | 9 | isoechoic echotexture | ImagingFindings | | 10 | muscle | BodyPart | | 11 | internal color flow | ImagingFindings | | 12 | benign fibrous tissue | ImagingFindings | | 13 | lipoma | Disease_Syndrome_Disorder | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_rd_ner_wip_greedy_biobert| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on Dataset annotated by John Snow Labs ## Benchmarking ```bash label tp fp fn prec rec f1 B-Units 253 7 11 0.9730769 0.9583333 0.9656488 B-Medical_Device 382 109 74 0.7780040 0.8377193 0.8067581 B-BodyPart 2645 347 276 0.8840241 0.9055118 0.8946389 I-BodyPart 645 142 135 0.819568 0.8269231 0.8232291 B-Imaging_Technique 137 36 33 0.7919075 0.8058823 0.7988338 B-Procedure 260 93 130 0.7365439 0.6666667 0.6998653 B-Direction 1573 136 123 0.9204213 0.9274764 0.9239353 I-ImagingTest 30 9 32 0.7692308 0.4838709 0.5940594 I-Test_Result 2 0 0 1 1 1 B-Measurements 452 24 30 0.9495798 0.9377593 0.9436326 B-ImagingFindings 1929 679 542 0.7396472 0.7806556 0.7595984 B-Test 146 17 49 0.8957055 0.7487179 0.8156425 B-ManualFix 2 0 2 1 0.5 0.6666667 I-Procedure 147 91 106 0.6176470 0.5810277 0.598778 I-Imaging_Technique 75 63 26 0.5434782 0.7425743 0.6276151 I-Measurements 45 3 6 0.9375 0.8823529 0.9090909 B-ImagingTest 328 36 85 0.9010989 0.7941888 0.8442728 I-Test 26 9 34 0.7428571 0.4333333 0.5473684 I-Symptom 138 62 142 0.69 0.4928571 0.575 I-ImagingFindings 1348 617 662 0.6860051 0.6706468 0.678239 B-Disease_Syndrome_Disorder 1068 298 243 0.7818448 0.8146453 0.7979080 B-Symptom 523 110 190 0.8262243 0.7335203 0.7771174 I-Disease_Syndrome_Disorder 377 168 171 0.6917431 0.6879562 0.6898445 I-Medical_Device 369 72 62 0.8367347 0.8561485 0.8463302 I-Direction 352 38 41 0.9025641 0.8956743 0.899106 Macro-average 13272 3200 3313 0.7195612 0.6489194 0.6824170 Micro-average 13272 3200 3313 0.8057309 0.8002412 0.8029767 ``` --- layout: model title: Legal Power of attorney Clause Binary Classifier (md) author: John Snow Labs name: legclf_power_of_attorney_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `power-of-attorney` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `power-of-attorney` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_power_of_attorney_md_en_1.0.0_3.0_1669376514743.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_power_of_attorney_md_en_1.0.0_3.0_1669376514743.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_power_of_attorney_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[power-of-attorney]| |[other]| |[other]| |[power-of-attorney]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_power_of_attorney_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support other 0.89 1.00 0.94 39 power-of-attorney 1.00 0.79 0.88 24 accuracy 0.92 63 macro avg 0.94 0.90 0.91 63 weighted avg 0.93 0.92 0.92 63 ``` --- layout: model title: ALBERT Large CoNNL-03 NER Pipeline author: John Snow Labs name: albert_large_token_classifier_conll03_pipeline date: 2022-04-23 tags: [open_source, ner, token_classifier, albert, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [albert_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/albert_large_token_classifier_conll03_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_large_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650710898514.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_large_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650710898514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_squad2_next_word_generator_squad","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_squad2_next_word_generator_squad","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_squad2_next_word_generator_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|148.1 MB| ## References - https://huggingface.co/allenai/t5-small-squad2-next-word-generator-squad --- layout: model title: Part of Speech for Japanese author: John Snow Labs name: pos_ud_gsd date: 2021-03-09 tags: [part_of_speech, open_source, japanese, pos_ud_gsd, ja] task: Part of Speech Tagging language: ja edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - NOUN - ADP - VERB - SCONJ - AUX - PUNCT - PART - DET - NUM - ADV - PRON - ADJ - PROPN - CCONJ - SYM - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ja_3.0.0_3.0_1615292368738.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ja_3.0.0_3.0_1615292368738.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_gsd", "ja") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['ジョンスノーラボからこんにちは！ ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_gsd", "ja") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("ジョンスノーラボからこんにちは！ ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""ジョンスノーラボからこんにちは！ ""] token_df = nlu.load('ja.pos.ud_gsd').predict(text) token_df ```

## Results ```bash token pos 0 ジョンス NOUN 1 ノ NOUN 2 ー NOUN 3 ラ NOUN 4 ボ NOUN 5 から ADP 6 こん NOUN 7 に ADP 8 ち NOUN 9 は ADP 10 ！ VERB ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|ja| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: roberta_qa_deepset_base_squad2_orkg_how_5e_05 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deepset-roberta-base-squad2-orkg-how-5e-05` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_how_5e_05_en_4.3.0_3.0_1674209532316.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepset_base_squad2_orkg_how_5e_05_en_4.3.0_3.0_1674209532316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_how_5e_05","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_how_5e_05","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_base_squad2_orkg_how_5e_05| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Moussab/deepset-roberta-base-squad2-orkg-how-5e-05 --- layout: model title: Pipeline for Adverse Drug Events author: John Snow Labs name: explain_clinical_doc_ade date: 2022-06-30 tags: [en, clinical, licensed, ade, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline for Adverse Drug Events (ADE) with `ner_ade_biobert`, `assertion_dl_biobert`, ,`classifierdl_ade_conversational_biobert`, and `re_ade_biobert` . It will classify the document, extract ADE and DRUG clinical entities, assign assertion status to ADE entities, and relate Drugs with their ADEs. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_ade_en_4.0.0_3.0_1656581944018.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_ade_en_4.0.0_3.0_1656581944018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("explain_clinical_doc_ade", "en", "clinical/models") res = pipeline.fullAnnotate("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""") ``` ```scala from sparknlp.pretrained import PretrainedPipeline val era_pipeline = new PretrainedPipeline("explain_clinical_doc_ade", "en", "clinical/models") val result = era_pipeline.fullAnnotate("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""")(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.clinical_ade").predict("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""") ```

## Results ```bash Class: True NER_Assertion: | | chunk | entitiy | assertion | |----|-------------------------|------------|-------------| | 0 | Lipitor | DRUG | - | | 1 | severe fatigue | ADE | Conditional | | 2 | voltaren | DRUG | - | | 3 | cramps | ADE | Conditional | Relations: | | chunk1 | entitiy1 | chunk2 | entity2 | relation | |----|-------------------------------|------------|-------------|---------|----------| | 0 | severe fatigue | ADE | Lipitor | DRUG | 1 | | 1 | cramps | ADE | Lipitor | DRUG | 0 | | 2 | severe fatigue | ADE | voltaren | DRUG | 0 | | 3 | cramps | ADE | voltaren | DRUG | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_ade| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|484.6 MB| ## Included Models - DocumentAssembler - TokenizerModel - BertEmbeddings - SentenceEmbeddings - ClassifierDLModel - MedicalNerModel - NerConverterInternalModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - NerConverterInternalModel - AssertionDLModel --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739275203.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739275203.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_quadruplet_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: Chinese BertForMaskedLM Base Cased model (from ptrsxu) author: John Snow Labs name: bert_embeddings_ptrsxu_base_chinese date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-chinese` is a Chinese model originally trained by `ptrsxu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_ptrsxu_base_chinese_zh_4.2.4_3.0_1670016435043.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_ptrsxu_base_chinese_zh_4.2.4_3.0_1670016435043.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_ptrsxu_base_chinese","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_ptrsxu_base_chinese","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_ptrsxu_base_chinese| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/ptrsxu/bert-base-chinese - https://aclanthology.org/2021.acl-long.330.pdf - https://dl.acm.org/doi/pdf/10.1145/3442188.3445922 --- layout: model title: Fast Neural Machine Translation Model from Spanish to English author: John Snow Labs name: opus_mt_es_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, es, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `es` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_es_en_xx_2.7.0_2.4_1609165183882.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_es_en_xx_2.7.0_2.4_1609165183882.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_es_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Uncased model (from roshnir) author: John Snow Labs name: bert_qa_multi_uncased_trained_squadv2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-uncased-trained-squadv2` is a English model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_multi_uncased_trained_squadv2_en_4.0.0_3.0_1657187851046.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_multi_uncased_trained_squadv2_en_4.0.0_3.0_1657187851046.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_multi_uncased_trained_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_multi_uncased_trained_squadv2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_multi_uncased_trained_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|626.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/bert-multi-uncased-trained-squadv2 - https://aclanthology.org/2020.acl-main.421.pdf%5D --- layout: model title: Tagalog Electra Embeddings (from jcblaise) author: John Snow Labs name: electra_embeddings_electra_tagalog_small_uncased_generator date: 2022-05-17 tags: [tl, open_source, electra, embeddings] task: Embeddings language: tl edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-tagalog-small-uncased-generator` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_small_uncased_generator_tl_3.4.4_3.0_1652786769151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_small_uncased_generator_tl_3.4.4_3.0_1652786769151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_tagalog_small_uncased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tl| |Size:|18.9 MB| |Case sensitive:|false| ## References - https://huggingface.co/jcblaise/electra-tagalog-small-uncased-generator - https://blaisecruz.com --- layout: model title: English asr_wav2vec2_base_100h_by_facebook TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_by_facebook date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_by_facebook` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_by_facebook_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_by_facebook_en_4.2.0_3.0_1664038682928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_by_facebook_en_4.2.0_3.0_1664038682928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_by_facebook', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_by_facebook", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_by_facebook| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-2` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_2_en_4.3.0_3.0_1672767456835.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_2_en_4.3.0_3.0_1672767456835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|243.9 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-2 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10_en_4.3.0_3.0_1674215893984.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10_en_4.3.0_3.0_1674215893984.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_64_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|419.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-10 --- layout: model title: Named Entity Recognition Profiling (Clinical) author: John Snow Labs name: ner_profiling_clinical date: 2022-01-18 tags: [ner, ner_profiling, clinical, en, licensed] task: [Named Entity Recognition, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `embeddings_clinical`. It has been updated by adding new clinical NER model and NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `ner_ade_clinical`, `ner_posology_greedy`, `ner_risk_factors`, `jsl_ner_wip_clinical`, `ner_human_phenotype_gene_clinical`, `jsl_ner_wip_greedy_clinical`, `ner_cellular`, `ner_cancer_genetics`, `jsl_ner_wip_modifier_clinical`, `ner_drugs_greedy`, `ner_deid_sd_large`, `ner_diseases`, `nerdl_tumour_demo`, `ner_deid_subentity_augmented`, `ner_jsl_enriched`, `ner_genetic_variants`, `ner_bionlp`, `ner_measurements_clinical`, `ner_diseases_large`, `ner_radiology`, `ner_deid_augmented`, `ner_anatomy`, `ner_chemprot_clinical`, `ner_posology_experimental`, `ner_drugs`, `ner_deid_sd`, `ner_posology_large`, `ner_deid_large`, `ner_posology`, `ner_deidentify_dl`, `ner_deid_enriched`, `ner_bacterial_species`, `ner_drugs_large`, `ner_clinical_large`, `jsl_rd_ner_wip_greedy_clinical`, `ner_medmentions_coarse`, `ner_radiology_wip_clinical`, `ner_clinical`, `ner_chemicals`, `ner_deid_synthetic`, `ner_events_clinical`, `ner_posology_small`, `ner_anatomy_coarse`, `ner_human_phenotype_go_clinical`, `ner_jsl_slim`, `ner_jsl`, `ner_jsl_greedy`, `ner_events_admission_clinical`, `ner_chexpert` . {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.2.Pretrained_NER_Profiling_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_3.3.1_2.4_1642496753293.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_3.3.1_2.4_1642496753293.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline('ner_profiling_clinical', 'en', 'clinical/models') result = ner_profiling_pipeline.annotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline('ner_profiling_clinical', 'en', 'clinical/models') val result = ner_profiling_pipeline.annotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_clinical").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ```

## Results ```bash ******************** ner_jsl Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('subsequent', 'Modifier'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Diabetes'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Communicable_Disease'), ('obesity', 'Obesity'), ('body mass index', 'Symptom'), ('33.5 kg/m2', 'Weight'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_diseases_large Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('vomiting', 'Disease')] ******************** ner_radiology Model Results ******************** [('gestational diabetes mellitus', 'Disease_Syndrome_Disorder'), ('type two diabetes mellitus', 'Disease_Syndrome_Disorder'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('acute hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Disease_Syndrome_Disorder'), ('body', 'BodyPart'), ('mass index', 'Symptom'), ('BMI', 'Test'), ('33.5', 'Measurements'), ('kg/m2', 'Units'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus', 'PROBLEM'), ('T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'PROBLEM'), ('BMI', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_medmentions_coarse Model Results ******************** [('female', 'Organism_Attribute'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('T2DM', 'Disease_or_Syndrome'), ('HTG-induced pancreatitis', 'Disease_or_Syndrome'), ('associated with', 'Qualitative_Concept'), ('acute hepatitis', 'Disease_or_Syndrome'), ('obesity', 'Disease_or_Syndrome'), ('body mass index', 'Clinical_Attribute'), ('BMI', 'Clinical_Attribute'), ('polyuria', 'Sign_or_Symptom'), ('polydipsia', 'Sign_or_Symptom'), ('poor appetite', 'Sign_or_Symptom'), ('vomiting', 'Sign_or_Symptom')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_clinical| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|2.4 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: Part of Speech for Swedish author: John Snow Labs name: pos_talbanken date: 2021-03-23 tags: [sv, open_source] supported: true task: Part of Speech Tagging language: sv edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_talbanken_sv_2.7.5_2.4_1616511099635.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_talbanken_sv_2.7.5_2.4_1616511099635.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_talbanken", "sv")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([["' Medicinsk bildtolk ' också skall fungera som hjälpmedel för läkaren att klarlägga sjukdomsbilden utan att patienten behöver säga ett ord ."]], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_talbanken", "sv") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer , pos)) val data = Seq(" Medicinsk bildtolk " också skall fungera som hjälpmedel för läkaren att klarlägga sjukdomsbilden utan att patienten behöver säga ett ord .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""' Medicinsk bildtolk ' också skall fungera som hjälpmedel för läkaren att klarlägga sjukdomsbilden utan att patienten behöver säga ett ord .""] token_df = nlu.load('sv.pos.talbanken').predict(text) token_df ```

## Results ```bash +---------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ |text |result | +---------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ |' Medicinsk bildtolk ' också skall fungera som hjälpmedel för läkaren att klarlägga sjukdomsbilden utan att patienten behöver säga ett ord . |[PUNCT, ADJ, NOUN, PUNCT, ADV, AUX, VERB, SCONJ, NOUN, ADP, NOUN, PART, VERB, NOUN, ADP, SCONJ, NOUN, AUX, VERB, DET, NOUN, PUNCT]| +---------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_talbanken| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|sv| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.88 | 0.89 | 0.89 | 1826 | | ADP | 0.96 | 0.96 | 0.96 | 2298 | | ADV | 0.91 | 0.87 | 0.89 | 1528 | | AUX | 0.91 | 0.93 | 0.92 | 1021 | | CCONJ | 0.95 | 0.94 | 0.94 | 791 | | DET | 0.92 | 0.95 | 0.93 | 1015 | | INTJ | 1.00 | 0.33 | 0.50 | 3 | | NOUN | 0.94 | 0.95 | 0.95 | 4711 | | NUM | 0.98 | 0.96 | 0.97 | 357 | | PART | 0.93 | 0.94 | 0.94 | 406 | | PRON | 0.94 | 0.91 | 0.92 | 1449 | | PROPN | 0.88 | 0.83 | 0.85 | 243 | | PUNCT | 0.97 | 0.98 | 0.98 | 2104 | | SCONJ | 0.86 | 0.82 | 0.84 | 491 | | SYM | 0.50 | 1.00 | 0.67 | 1 | | VERB | 0.90 | 0.90 | 0.90 | 2142 | | accuracy | | | 0.93 | 20386 | | macro avg | 0.90 | 0.89 | 0.88 | 20386 | | weighted avg | 0.93 | 0.93 | 0.93 | 20386 | ``` --- layout: model title: Detect Adverse Drug Events (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_ade date: 2021-09-30 tags: [adverse, ade, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.2.2 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse reactions of drugs in reviews, tweets, and medical text using the pretrained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `DRUG`, `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_en_3.2.2_2.4_1633008677011.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_en_3.2.2_2.4_1633008677011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_ade", "en", "clinical/models")\ .setInputCols("token", "document")\ .setOutputCol("ner")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier, ner_converter]) p_model = pipeline.fit(spark.createDataFrame(pd.DataFrame({'text': ['']}))) test_sentence = """Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""" result = p_model.transform(spark.createDataFrame(pd.DataFrame({'text': [test_sentence]}))) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_ade", "en", "clinical/models") .setInputCols(Array("token", "document")) .setOutputCol("ner") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("document","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier, ner_converter)) val data = Seq("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_ade").predict("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import pyspark.sql.functions as F document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = finance.NerModel.pretrained("finner_orgs_prods_alias","en","finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") finassertion = finance.AssertionDLModel.pretrained("finassertion_negation", "en", "finance/models")\ .setInputCols(["sentence", "ner_chunk", "embeddings"])\ .setOutputCol("finlabel") pipe = nlp.Pipeline(stages = [ document_assembler, sentence_detector, tokenizer, embeddings, ner, ner_converter, finassertion]) text = "Gradio INC will not be entering into a joint agreement with Hugging Face, Inc." sdf = spark.createDataFrame([[text]]).toDF("text") res = pipe.fit(sdf).transform(sdf) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.finlabel.result)).alias("cols"))\ .select(F.expr("cols['0']").alias("ner_chunk"), F.expr("cols['1']").alias("assertion")).show(200, truncate=100) ```

## Results ```bash +-----------------+---------+ | ner_chunk|assertion| +-----------------+---------+ | Gradio INC| negative| |Hugging Face, Inc| positive| +-----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finassertion_negation| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|2.2 MB| ## References In-house annotated legal sentences ## Benchmarking ```bash label tp fp fn prec rec f1 negative 26 0 1 1.0 0.962963 0.9811321 positive 38 1 0 0.974359 1.0 0.987013 Macro-average 641 1 1 0.9871795 0.9814815 0.9843222 Micro-average 0.9846154 0.9846154 0.9846154 ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-256-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0_en_4.0.0_3.0_1657192274652.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0_en_4.0.0_3.0_1657192274652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|384.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-256-finetuned-squad-seed-0 --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2_en_4.0.0_3.0_1655732129289.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2_en_4.0.0_3.0_1655732129289.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_256d_seed_2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|427.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-2 --- layout: model title: Legal Services Clause Binary Classifier author: John Snow Labs name: legclf_services_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `services` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `services` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_services_clause_en_1.0.0_3.2_1660123991970.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_services_clause_en_1.0.0_3.2_1660123991970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_services_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[services]| |[other]| |[other]| |[services]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_services_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.95 0.96 78 services 0.89 0.94 0.91 34 accuracy - - 0.95 112 macro-avg 0.93 0.94 0.94 112 weighted-avg 0.95 0.95 0.95 112 ``` --- layout: model title: Icelandic NER Pipeline author: John Snow Labs name: roberta_token_classifier_icelandic_ner_pipeline date: 2022-06-25 tags: [open_source, ner, token_classifier, roberta, icelandic, is] task: Named Entity Recognition language: is edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_token_classifier_icelandic_ner](https://nlp.johnsnowlabs.com/2021/12/06/roberta_token_classifier_icelandic_ner_is.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_icelandic_ner_pipeline_is_4.0.0_3.0_1656122302435.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_icelandic_ner_pipeline_is_4.0.0_3.0_1656122302435.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------+------------+ |chunk |ner_label | +----------------+------------+ |Peter Fergusson |Person | |New York |Location | |október 2011 |Date | |Tesla Motor |Organization| |100K $ |Money | +----------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_icelandic_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|is| |Size:|457.5 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from yohein) author: John Snow Labs name: distilbert_qa_yohein_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `yohein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_yohein_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773298911.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_yohein_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773298911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_yohein_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_yohein_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_yohein_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/yohein/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Detect biological concepts (biobert) author: John Snow Labs name: ner_bionlp_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_bionlp_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_bionlp_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_pipeline_en_4.3.0_3.2_1679313010526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_pipeline_en_4.3.0_3.2_1679313010526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_bionlp_biobert_pipeline", "en", "clinical/models") text = '''Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_bionlp_biobert_pipeline", "en", "clinical/models") val text = "Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay" val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.bionlp_biobert.pipeline").predict("""Both the erbA IRES and the erbA/myb virus constructs transformed erythroid cells after infection of bone marrow or blastoderm cultures. The erbA/myb IRES virus exhibited a 5-10-fold higher transformed colony forming efficiency than the erbA IRES virus in the blastoderm assay""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:-----------------------|-------------:| | 0 | erbA | 9 | 12 | Gene_or_gene_product | 1 | | 1 | IRES | 14 | 17 | Organism | 0.754 | | 2 | virus | 36 | 40 | Organism | 0.9999 | | 3 | erythroid cells | 65 | 79 | Cell | 0.99855 | | 4 | bone | 100 | 103 | Multi-tissue_structure | 0.9794 | | 5 | marrow | 105 | 110 | Multi-tissue_structure | 0.9631 | | 6 | blastoderm cultures | 115 | 133 | Cell | 0.9868 | | 7 | IRES virus | 149 | 158 | Organism | 0.99985 | | 8 | erbA | 236 | 239 | Gene_or_gene_product | 0.9977 | | 9 | IRES virus | 241 | 250 | Organism | 0.9911 | | 10 | blastoderm | 259 | 268 | Cell | 0.9941 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bionlp_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Pipeline to Detect Radiology Related Entities author: John Snow Labs name: ner_radiology_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_radiology](https://nlp.johnsnowlabs.com/2021/03/31/ner_radiology_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_radiology_pipeline_en_4.3.0_3.2_1678865918152.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_radiology_pipeline_en_4.3.0_3.2_1678865918152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_radiology_pipeline", "en", "clinical/models") text = '''Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_radiology_pipeline", "en", "clinical/models") val text = "Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.radiology.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------------------------|--------:|------:|:--------------------------|-------------:| | 0 | Bilateral breast | 0 | 15 | BodyPart | 0.945 | | 1 | ultrasound | 17 | 26 | ImagingTest | 0.6734 | | 2 | ovoid mass | 78 | 87 | ImagingFindings | 0.6095 | | 3 | 0.5 x 0.5 x 0.4 | 113 | 127 | Measurements | 0.98158 | | 4 | cm | 129 | 130 | Units | 0.9696 | | 5 | anteromedial aspect of the left shoulder | 163 | 202 | BodyPart | 0.750517 | | 6 | mass | 210 | 213 | ImagingFindings | 0.9711 | | 7 | isoechoic echotexture | 228 | 248 | ImagingFindings | 0.80105 | | 8 | muscle | 266 | 271 | BodyPart | 0.7963 | | 9 | internal color flow | 294 | 312 | ImagingFindings | 0.477233 | | 10 | benign fibrous tissue | 334 | 354 | ImagingFindings | 0.524067 | | 11 | lipoma | 361 | 366 | Disease_Syndrome_Disorder | 0.6081 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_radiology_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Translate Italian to English Pipeline author: John Snow Labs name: translate_it_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, it, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `it` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_it_en_xx_2.7.0_2.4_1609689489920.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_it_en_xx_2.7.0_2.4_1609689489920.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_it_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_it_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.it.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_it_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Portuguese BertForTokenClassification Cased model (from pucpr) author: John Snow Labs name: bert_token_classifier_clinicalnerpt_diagnostic date: 2022-11-30 tags: [pt, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: pt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `clinicalnerpt-diagnostic` is a Portuguese model originally trained by `pucpr`. ## Predicted Entities `DiagnosticProcedure` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_diagnostic_pt_4.2.4_3.0_1669822359730.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_diagnostic_pt_4.2.4_3.0_1669822359730.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_diagnostic","pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_diagnostic","pt") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_clinicalnerpt_diagnostic| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|pt| |Size:|665.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/pucpr/clinicalnerpt-diagnostic - https://www.aclweb.org/anthology/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/SemClinBr - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `recipe_triplet_recipes-roberta-base_EASY_TIMESTEP_squadv2_epochs_3` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3_en_4.3.0_3.0_1674212108351.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3_en_4.3.0_3.0_1674212108351.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_recipe_triplet_recipes_base_easy_timestep_squadv2_epochs_3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/recipe_triplet_recipes-roberta-base_EASY_TIMESTEP_squadv2_epochs_3 --- layout: model title: Lemmatizer (Dutch, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, nl] task: Lemmatization language: nl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Dutch Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_nl_3.4.1_3.0_1646316580197.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_nl_3.4.1_3.0_1646316580197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------+ |result | +--------------------------------+ |[Je, bent, niet, beter, dan, ik]| +--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|nl| |Size:|2.5 MB| --- layout: model title: Detect Problems, Tests and Treatments (ner_clinical_large) author: John Snow Labs name: ner_clinical date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terms. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `PROBLEM`, `TEST`, `TREATMENT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_en_3.0.0_3.0_1617208419368.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_en_3.0.0_3.0_1617208419368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.""") ```

## Results ```bash +-----------------------------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------------------------+---------+ |the G-protein-activated inwardly rectifying potassium (GIRK|TREATMENT| |the genomicorganization |TREATMENT| |a candidate gene forType II diabetes mellitus |PROBLEM | |byapproximately |TREATMENT| |single nucleotide polymorphisms |TREATMENT| |aVal366Ala substitution |TREATMENT| |an 8 base-pair |TREATMENT| |insertion/deletion |PROBLEM | |Ourexpression studies |TEST | |the transcript in various humantissues |PROBLEM | |fat andskeletal muscle |PROBLEM | |furtherstudies |PROBLEM | |the KCNJ9 protein |TREATMENT| |evaluation |TEST | |Type II diabetes |PROBLEM | |the treatment |TREATMENT| |breast cancer |PROBLEM | |the standard therapy |TREATMENT| |anthracyclines |TREATMENT| |taxanes |TREATMENT| +-----------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on augmented 2010 i2b2 challenge data with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | I-TREATMENT | 6625 | 1187 | 1329 | 0.848054 | 0.832914 | 0.840416 | | 1 | I-PROBLEM | 15142 | 1976 | 2542 | 0.884566 | 0.856254 | 0.87018 | | 2 | B-PROBLEM | 11005 | 1065 | 1587 | 0.911765 | 0.873968 | 0.892466 | | 3 | I-TEST | 6748 | 923 | 1264 | 0.879677 | 0.842237 | 0.86055 | | 4 | B-TEST | 8196 | 942 | 1029 | 0.896914 | 0.888455 | 0.892665 | | 5 | B-TREATMENT | 8271 | 1265 | 1073 | 0.867345 | 0.885167 | 0.876165 | | 6 | Macro-average | 55987 | 7358 | 8824 | 0.881387 | 0.863166 | 0.872181 | | 7 | Micro-average | 55987 | 7358 | 8824 | 0.883842 | 0.86385 | 0.873732 | ``` --- layout: model title: Spanish ElectraForQuestionAnswering Small model (from mrm8488) author: John Snow Labs name: electra_qa_electricidad_small_finetuned_squadv1 date: 2022-06-22 tags: [es, open_source, electra, question_answering] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electricidad-small-finetuned-squadv1-es` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_electricidad_small_finetuned_squadv1_es_4.0.0_3.0_1655921719427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_electricidad_small_finetuned_squadv1_es_4.0.0_3.0_1655921719427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_electricidad_small_finetuned_squadv1","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_electricidad_small_finetuned_squadv1","es") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.electra.small").predict("""¿Cuál es mi nombre?|||"Mi nombre es Clara y vivo en Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_electricidad_small_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|51.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/electricidad-small-finetuned-squadv1-es - https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1 --- layout: model title: Smaller BERT Sentence Embeddings (L-4_H-512_A-8) author: John Snow Labs name: sent_small_bert_L4_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L4_512_en_2.6.0_2.4_1598350568942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L4_512_en_2.6.0_2.4_1598350568942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L4_512", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L4_512", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L4_512').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L4_512_embeddings sentence [0.2255069762468338, 0.14144930243492126, 0.67... I hate cancer [-0.5351444482803345, 0.36734339594841003, 0.1... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L4_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1 --- layout: model title: Finnish asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP TFWav2Vec2ForCTC from Finnish-NLP author: John Snow Labs name: asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP` is a Finnish model originally trained by Finnish-NLP. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP_fi_4.2.0_3.0_1664025529159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP_fi_4.2.0_3.0_1664025529159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_1b_finnish_lm_by_Finnish_NLP| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|3.6 GB| --- layout: model title: Pipeline to Detect Adverse Drug Events (healthcare) author: John Snow Labs name: ner_ade_healthcare_pipeline date: 2022-03-22 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_ade_healthcare](https://nlp.johnsnowlabs.com/2021/04/01/ner_ade_healthcare_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_ade_healthcare_pipeline_en_3.4.1_3.0_1647944180015.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_ade_healthcare_pipeline_en_3.4.1_3.0_1647944180015.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_ade_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Been taking Lipitor for 15 years, have experienced severe fatigue a lot!!!. Doctor moved me to voltaren 2 months ago, so far, have only experienced cramps") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_ade_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Been taking Lipitor for 15 years, have experienced severe fatigue a lot!!!. Doctor moved me to voltaren 2 months ago, so far, have only experienced cramps") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.healthcare_ade.pipeline").predict("""Been taking Lipitor for 15 years, have experienced severe fatigue a lot!!!. Doctor moved me to voltaren 2 months ago, so far, have only experienced cramps""") ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |Lipitor |DRUG | |severe fatigue|ADE | |voltaren |DRUG | |cramps |ADE | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ade_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.5 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Legal Participations Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_participations_bert date: 2023-03-05 tags: [en, legal, classification, clauses, participations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Participations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Participations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_participations_bert_en_1.0.0_3.0_1678050674961.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_participations_bert_en_1.0.0_3.0_1678050674961.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_participations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Participations]| |[Other]| |[Other]| |[Participations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_participations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.93 0.96 0.95 73 Participations 0.94 0.91 0.92 53 accuracy - - 0.94 126 macro-avg 0.94 0.93 0.93 126 weighted-avg 0.94 0.94 0.94 126 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2_en_4.3.0_3.0_1674215491724.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2_en_4.3.0_3.0_1674215491724.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_512_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|432.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-2 --- layout: model title: Stopwords Remover for Ligurian language (162 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, lij, open_source] task: Stop Words Removal language: lij edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_lij_3.4.1_3.0_1646653713114.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_lij_3.4.1_3.0_1646653713114.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+ |result | +-----------------+ |[Unde, marina, ?]| +-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|lij| |Size:|1.8 KB| --- layout: model title: English BertForQuestionAnswering model (from haddadalwi) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-finetuned-squad-finetuned-islamic-squad` is a English model orginally trained by `haddadalwi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad_en_4.0.0_3.0_1654537141717.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad_en_4.0.0_3.0_1654537141717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.large_uncased.by_haddadalwi").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_finetuned_squad_finetuned_islamic_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/haddadalwi/bert-large-uncased-whole-word-masking-finetuned-squad-finetuned-islamic-squad --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes author: John Snow Labs name: sbiobertresolve_icd10cm_augmented_billable_hcc date: 2021-11-01 tags: [icd10cm, hcc, entity_resolution, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.1 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings and it supports 7-digit codes with HCC status. It has been updated by dropping the invalid codes that exist in the previous versions. In the result, look for the `all_k_aux_labels` parameter in the metadata to get HCC status. The HCC status can be divided to get further information: `billable status`, `hcc status`, and `hcc score`. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_3.3.1_2.4_1635784379929.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_3.3.1_2.4_1635784379929.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_icd10cm_augmented_billable_hcc``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. ```PROBLEM``` set in ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... c2doc = Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sentence_embeddings") icd_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_icd10cm_augmented_billable_hcc", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sentence_embeddings"]) \ .setOutputCol("icd10cm_code")\ .setDistanceFunction("EUCLIDEAN") resolver_pipeline = Pipeline( stages = [ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, clinical_ner, ner_converter_icd, c2doc, sbert_embedder, icd_resolver ]) data_ner = spark.createDataFrame([["A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection."]]).toDF("text") results = resolver_pipeline.fit(data_ner).transform(data_ner) ``` ```scala ... val chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val icd10_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_icd10cm_augmented_billable_hcc","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, icd10_resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd10cm.augmented_billable").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection.""") ```

## Results ```bash +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk| entity|icd10cm_code| resolutions| all_codes| billable_hcc| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O2441|gestational diabetes mellitus:::postpartum gestational diabetes mel...| O2441:::O2443:::Z8632:::Z875:::O2431:::O2411:::O244:::O241:::O2481|0||0||0:::0||0||0:::1||0||0:::0||0||0:::0||0||0:::0||0||0:::0||0||0...| |subsequent type two diabetes mellitus|PROBLEM| O2411|pre-existing type 2 diabetes mellitus:::disorder associated with ty...|O2411:::E118:::E11:::E139:::E119:::E113:::E1144:::Z863:::Z8639:::E1...|0||0||0:::1||1||18:::0||0||0:::1||1||19:::1||1||19:::0||0||0:::1||1...| | T2DM|PROBLEM| E11|t2dm [type 2 diabetes mellitus]:::tndm2:::t2 category:::sma2:::nf2:...|E11:::P702:::C801:::G121:::Q850:::C779:::C509:::C439:::E723:::C5700...|0||0||0:::1||0||0:::1||1||12:::1||1||72:::0||0||0:::1||1||10:::0||0...| | HTG-induced pancreatitis|PROBLEM| K8520|alcohol-induced pancreatitis:::pancreatitis:::drug induced acute pa...|K8520:::K859:::K853:::K8590:::K85:::F102:::K858:::K8591:::K852:::K8...|1||0||0:::0||0||0:::0||0||0:::1||0||0:::0||0||0:::0||0||0:::0||0||0...| | acute hepatitis|PROBLEM| K720|acute hepatitis:::acute hepatitis a:::acute infectious hepatitis:::...|K720:::B15:::B179:::B172:::Z0389:::B159:::B150:::B16:::K752:::K712:...|0||0||0:::0||0||0:::1||0||0:::1||0||0:::1||0||0:::1||0||0:::1||0||0...| | obesity|PROBLEM| E669|obesity:::abdominal obesity:::obese:::central obesity:::overweight ...|E669:::E668:::Z6841:::Q130:::E66:::E6601:::Z8639:::E349:::H3550:::Z...|1||0||0:::1||0||0:::1||1||22:::1||0||0:::0||0||0:::1||1||22:::1||0|...| | a body mass index|PROBLEM| Z6841|finding of body mass index:::observation of body mass index:::mass ...|Z6841:::E669:::R229:::Z681:::R223:::R221:::Z68:::R222:::R220:::R418...|1||1||22:::1||0||0:::1||0||0:::1||0||0:::0||0||0:::1||0||0:::0||0||...| | polyuria|PROBLEM| R35|polyuria:::polyuric state:::polyuric state (disorder):::hematuria::...|R35:::R358:::E232:::R31:::R350:::R8299:::N401:::E723:::O048:::R300:...|0||0||0:::1||0||0:::1||1||23:::0||0||0:::1||0||0:::0||0||0:::1||0||...| | polydipsia|PROBLEM| R631|polydipsia:::psychogenic polydipsia:::primary polydipsia:::psychoge...|R631:::F6389:::E232:::F639:::O40:::G475:::M7989:::R632:::R061:::H53...|1||0||0:::1||1||nan:::1||1||23:::1||1||nan:::0||0||0:::0||0||0:::1|...| | poor appetite|PROBLEM| R630|poor appetite:::poor feeding:::bad taste in mouth:::unpleasant tast...|R630:::P929:::R438:::R432:::E86:::R196:::F520:::Z724:::R0689:::Z768...|1||0||0:::1||0||0:::1||0||0:::1||0||0:::0||0||0:::1||0||0:::1||0||0...| | vomiting|PROBLEM| R111|vomiting:::intermittent vomiting:::vomiting symptoms:::periodic vom...| R111:::R11:::R1110:::G43A1:::P921:::P9209:::G43A:::R1113:::R110|0||0||0:::0||0||0:::1||0||0:::1||1||nan:::1||0||0:::1||0||0:::0||0|...| | a respiratory tract infection|PROBLEM| J988|respiratory tract infection:::upper respiratory tract infection:::b...|J988:::J069:::A499:::J22:::J209:::Z593:::T17:::J0410:::Z1383:::J189...|1||0||0:::1||0||0:::1||0||0:::1||0||0:::1||0||0:::1||0||0:::0||0||0...| +-------------------------------------+-------+------------+----------------------------------------------------------------------+----------------------------------------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented_billable_hcc| |Compatibility:|Healthcare NLP 3.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on 01 November 2021 ICD10CM Dataset. --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465525 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465525` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465525_en_4.0.0_3.0_1655987202658.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465525_en_4.0.0_3.0_1655987202658.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465525","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465525","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465525.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465525| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465525 --- layout: model title: Bangla BertForMaskedLM Base Cased model (from sagorsarker) author: John Snow Labs name: bert_embeddings_bangla_base date: 2022-12-02 tags: [bn, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: bn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bangla-bert-base` is a Bangla model originally trained by `sagorsarker`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_base_bn_4.2.4_3.0_1670015550585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_base_bn_4.2.4_3.0_1670015550585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_bangla_base","bn") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_bangla_base","bn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bangla_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|617.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/sagorsarker/bangla-bert-base - https://github.com/sagorbrur/bangla-bert - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://oscar-corpus.com/ - https://dumps.wikimedia.org/bnwiki/latest/ - https://github.com/sagorbrur/bnlp - https://github.com/sagorbrur/bangla-bert - https://github.com/google-research/bert - https://twitter.com/mapmeld - https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP - https://github.com/sagorbrur/bangla-bert/blob/master/notebook/bangla-bert-evaluation-classification-task.ipynb - https://github.com/sagorbrur/bangla-bert/tree/master/evaluations/wikiann - https://arxiv.org/abs/2012.14353 - https://arxiv.org/abs/2104.08613 - https://arxiv.org/abs/2107.03844 - https://arxiv.org/abs/2101.00204 - https://github.com/sagorbrur - https://www.tensorflow.org/tfrc - https://github.com/google-research/bert --- layout: model title: TREC(50) Question Classifier author: John Snow Labs name: classifierdl_use_trec50 date: 2021-01-08 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [classifier, text_classification, en, open_source] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify open-domain, fact-based questions into sub categories of the following broad semantic categories: Abbreviation, Description, Entities, Human Beings, Locations or Numeric Values. ## Predicted Entities ``ENTY_animal``, ``ENTY_body``, ``ENTY_color``, ``ENTY_cremat``, ``ENTY_currency``, ``ENTY_dismed``, ``ENTY_event``, ``ENTY_food``, ``ENTY_instru``, ``ENTY_lang``, ``ENTY_letter``, ``ENTY_other``, ``ENTY_plant``, ``ENTY_product``, ``ENTY_religion``, ``ENTY_sport``, ``ENTY_substance``, ``ENTY_symbol``, ``ENTY_techmeth``, ``ENTY_termeq``, ``ENTY_veh``, ``ENTY_word``, ``DESC_def``, ``DESC_desc``, ``DESC_manner``, ``DESC_reason``, ``HUM_gr``, ``HUM_ind``, ``HUM_title``, ``HUM_desc``, ``LOC_city``, ``LOC_country``, ``LOC_mount``, ``LOC_other``, ``LOC_state``, ``NUM_code``, ``NUM_count``, ``NUM_date``, ``NUM_dist``, ``NUM_money``, ``NUM_ord``, ``NUM_other``, ``NUM_period``, ``NUM_perc``, ``NUM_speed``, ``NUM_temp``, ``NUM_volsize``, ``NUM_weight``, ``ABBR_abb``, ``ABBR_exp``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_EN_TREC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_EN_TREC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec50_en_2.7.1_2.4_1610118328412.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec50_en_2.7.1_2.4_1610118328412.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained(lang="en") \ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") document_classifier = ClassifierDLModel.pretrained('classifierdl_use_trec50', 'en') \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") nlpPipeline = Pipeline(stages=[documentAssembler, use, document_classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate('When did the construction of stone circles begin in the UK?') ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val use = UniversalSentenceEncoder.pretrained(lang="en") .setInputCols(Array("document")) .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_use_trec50", "en") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, use, document_classifier)) val data = Seq("When did the construction of stone circles begin in the UK?").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""When did the construction of stone circles begin in the UK?"""] trec50_df = nlu.load('en.classify.trec50.use').predict(text, output_level = "document") trec50_df[["document", "trec50"]] ```

## Results ```bash +------------------------------------------------------------------------------------------------+------------+ |document |class | +------------------------------------------------------------------------------------------------+------------+ |When did the construction of stone circles begin in the UK? | NUM_date | +------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_trec50| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| ## Data Source This model is trained on the 50 class version of the TREC dataset. http://search.r-project.org/library/textdata/html/dataset_trec.html --- layout: model title: Legal Letters of credit Clause Binary Classifier author: John Snow Labs name: legclf_letters_of_credit_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `letters-of-credit` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `letters-of-credit` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_letters_of_credit_clause_en_1.0.0_3.2_1660122609389.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_letters_of_credit_clause_en_1.0.0_3.2_1660122609389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_letters_of_credit_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[letters-of-credit]| |[other]| |[other]| |[letters-of-credit]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_letters_of_credit_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support letters-of-credit 0.91 0.88 0.89 24 other 0.96 0.98 0.97 84 accuracy - - 0.95 108 macro-avg 0.94 0.93 0.93 108 weighted-avg 0.95 0.95 0.95 108 ``` --- layout: model title: Legal Delegation of duties Clause Binary Classifier author: John Snow Labs name: legclf_delegation_of_duties_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `delegation-of-duties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `delegation-of-duties` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_delegation_of_duties_clause_en_1.0.0_3.2_1660122337035.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_delegation_of_duties_clause_en_1.0.0_3.2_1660122337035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_delegation_of_duties_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[delegation-of-duties]| |[other]| |[other]| |[delegation-of-duties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_delegation_of_duties_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support delegation-of-duties 0.97 0.94 0.95 33 other 0.98 0.99 0.99 103 accuracy - - 0.98 136 macro-avg 0.97 0.96 0.97 136 weighted-avg 0.98 0.98 0.98 136 ``` --- layout: model title: Bangla RobertaForSequenceClassification Cased model (from neuralspace) author: John Snow Labs name: roberta_classifier_autotrain_citizen_nlu_bn_1370652766 date: 2022-12-09 tags: [bn, open_source, roberta, sequence_classification, classification, tensorflow] task: Text Classification language: bn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-citizen_nlu_bn-1370652766` is a Bangla model originally trained by `neuralspace`. ## Predicted Entities `ReportingMissingPets`, `EligibilityForBloodDonationCovidGap`, `ReportingPropertyTakeOver`, `IntentForBloodReceivalAppointment`, `EligibilityForBloodDonationSTD`, `InquiryForDoctorConsultation`, `InquiryOfCovidSymptoms`, `InquiryForVaccineCount`, `InquiryForCovidPrevention`, `InquiryForVaccinationRequirements`, `EligibilityForBloodDonationForPregnantWomen`, `ReportingCyberCrime`, `ReportingHitAndRun`, `ReportingTresspassing`, `InquiryofBloodDonationRequirements`, `ReportingMurder`, `ReportingVehicleAccident`, `ReportingMissingPerson`, `EligibilityForBloodDonationAgeLimit`, `ReportingAnimalPoaching`, `InquiryOfEmergencyContact`, `InquiryForQuarantinePeriod`, `ContactRealPerson`, `IntentForBloodDonationAppointment`, `ReportingMissingVehicle`, `InquiryForCovidRecentCasesCount`, `InquiryOfContact`, `StatusOfFIR`, `InquiryofVaccinationAgeLimit`, `InquiryForCovidTotalCasesCount`, `EligibilityForBloodDonationGap`, `InquiryofPostBloodDonationEffects`, `InquiryofPostBloodReceivalCareSchemes`, `EligibilityForBloodReceiversBloodGroup`, `EligitbilityForVaccine`, `InquiryOfLockdownDetails`, `ReportingSexualAssault`, `InquiryForVaccineCost`, `InquiryForCovidDeathCount`, `ReportingDrugConsumption`, `ReportingDrugTrafficing`, `InquiryofPostBloodDonationCertificate`, `ReportingDowry`, `ReportingChildAbuse`, `ReportingAnimalAbuse`, `InquiryofPostBloodReceivalEffects`, `Eligibility For BloodDonationWithComorbidities`, `InquiryOfTiming`, `InquiryForCovidActiveCasesCount`, `InquiryOfLocation`, `InquiryofPostBloodDonationCareSchemes`, `ReportingTheft`, `InquiryForTravelRestrictions`, `ReportingDomesticViolence`, `InquiryofBloodReceivalRequirements` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_citizen_nlu_bn_1370652766_bn_4.2.4_3.0_1670623640434.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_citizen_nlu_bn_1370652766_bn_4.2.4_3.0_1670623640434.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_citizen_nlu_bn_1370652766","bn") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_citizen_nlu_bn_1370652766","bn") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_autotrain_citizen_nlu_bn_1370652766| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|bn| |Size:|312.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/neuralspace/autotrain-citizen_nlu_bn-1370652766 --- layout: model title: Pipeline to Detect Radiology Entities, Assign Assertion Status and Find Relations author: John Snow Labs name: explain_clinical_doc_radiology date: 2023-04-20 tags: [licensed, clinical, en, ner, assertion, relation_extraction, radiology] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline for detecting posology entities with the `ner_radiology` NER model, assigning their assertion status with `assertion_dl_radiology` model, and extracting relations between posology-related terminology with `re_test_problem_finding` relation extraction model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_radiology_en_4.3.0_3.2_1682019248720.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_radiology_en_4.3.0_3.2_1682019248720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("explain_clinical_doc_radiology", "en", "clinical/models") text = """Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""" result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("explain_clinical_doc_radiology", "en", "clinical/models") val text = """Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""" val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.clinical_radiology.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash +----+------------------------------------------+---------------------------+ | | chunks | entities | |---:|:-----------------------------------------|:--------------------------| | 0 | Bilateral breast | BodyPart | | 1 | ultrasound | ImagingTest | | 2 | ovoid mass | ImagingFindings | | 3 | 0.5 x 0.5 x 0.4 | Measurements | | 4 | cm | Units | | 5 | anteromedial aspect of the left shoulder | BodyPart | | 6 | mass | ImagingFindings | | 7 | isoechoic echotexture | ImagingFindings | | 8 | muscle | BodyPart | | 9 | internal color flow | ImagingFindings | | 10 | benign fibrous tissue | ImagingFindings | | 11 | lipoma | Disease_Syndrome_Disorder | +----+------------------------------------------+---------------------------+ +----+-----------------------+---------------------------+-------------+ | | chunks | entities | assertion | |---:|:----------------------|:--------------------------|:------------| | 0 | ultrasound | ImagingTest | Confirmed | | 1 | ovoid mass | ImagingFindings | Confirmed | | 2 | mass | ImagingFindings | Confirmed | | 3 | isoechoic echotexture | ImagingFindings | Confirmed | | 4 | internal color flow | ImagingFindings | Negative | | 5 | benign fibrous tissue | ImagingFindings | Suspected | | 6 | lipoma | Disease_Syndrome_Disorder | Suspected | +----+-----------------------+---------------------------+-------------+ +---------+-----------------+-----------------------+---------------------------+------------+ |relation | entity1 | chunk1 | entity2 | chunk2 | |--------:|:----------------|:----------------------|:--------------------------|:-----------| | 1 | ImagingTest | ultrasound | ImagingFindings | ovoid mass | | 0 | ImagingFindings | benign fibrous tissue | Disease_Syndrome_Disorder | lipoma | +---------+-----------------+-----------------------+---------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_radiology| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - NerConverterInternalModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel --- layout: model title: English asr_Fine_Tuned_XLSR_English TFWav2Vec2ForCTC from Sania67 author: John Snow Labs name: asr_Fine_Tuned_XLSR_English date: 2022-09-26 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Fine_Tuned_XLSR_English` is a English model originally trained by Sania67. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Fine_Tuned_XLSR_English_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Fine_Tuned_XLSR_English_en_4.2.0_3.0_1664199621537.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Fine_Tuned_XLSR_English_en_4.2.0_3.0_1664199621537.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Fine_Tuned_XLSR_English", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Fine_Tuned_XLSR_English", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Fine_Tuned_XLSR_English| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_256 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-256` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_256_zh_4.2.4_3.0_1670021717667.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_256_zh_4.2.4_3.0_1670021717667.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_256","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_256","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_256| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|39.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-256 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Sentence Entity Resolver for Snomed Aux Concepts, INT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_auxConcepts_int language: en nav_key: models repository: clinical/models date: 2020-11-27 task: Entity Resolution edition: Healthcare NLP 2.6.4 spark_version: 2.4 tags: [clinical,entity_resolution,en] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model maps extracted medical entities to Snomed codes (with Morph Abnormality, Procedure, Substance, Physical Object, Body Structure concepts from INT version) using chunk embeddings. {:.h2_title} ## Predicted Entities Snomed Codes and their normalized definition with ``sbiobert_base_cased_mli`` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_int_en_2.6.4_2.4_1606235764318.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_int_en_2.6.4_2.4_1606235764318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") snomed_aux_int_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_snomed_auxConcepts_int","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_int_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val snomed_aux_int_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_auxConcepts_int","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_int_resolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 148439002| 0.2138|risk factors pres...|148439002:::42595...| |chronic renal ins...| 83|109| PROBLEM| 722403003| 0.8517|gastrointestinal ...|722403003:::13781...| | COPD| 113|116| PROBLEM|845101000000100| 0.0962|management of chr...|845101000000100::...| | gastritis| 120|128| PROBLEM| 711498001| 0.3398|magnetic resonanc...|711498001:::71771...| | TIA| 136|138| PROBLEM| 449758002| 0.1927|traumatic infarct...|449758002:::85844...| |a non-ST elevatio...| 182|202| PROBLEM| 1411000087101| 0.0823|ct of left knee::...|1411000087101:::3...| |Guaiac positive s...| 208|229| PROBLEM| 388507006| 0.0555|asparagus rast:::...|388507006:::71771...| |cardiac catheteri...| 295|317| TEST| 41976001| 0.9790|cardiac catheteri...|41976001:::705921...| | PTCA| 324|327|TREATMENT| 312644004| 0.0616|angioplasty of po...|312644004:::41507...| | mid LAD lesion| 332|345| PROBLEM| 91749005| 0.1399|structure of firs...|91749005:::917470...| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_snomed_auxConcepts_int | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.4 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on SNOMED (INT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: Translate North Germanic languages to English Pipeline author: John Snow Labs name: translate_gmq_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, gmq, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `gmq` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_gmq_en_xx_2.7.0_2.4_1609687798521.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_gmq_en_xx_2.7.0_2.4_1609687798521.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gmq_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gmq_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gmq.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gmq_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Chinese Bert Embeddings (Large, Roberta, Whole Word Masking) author: John Snow Labs name: bert_embeddings_chinese_roberta_wwm_ext_large date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-roberta-wwm-ext-large` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_ext_large_zh_3.4.2_3.0_1649668978927.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_wwm_ext_large_zh_3.4.2_3.0_1649668978927.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_wwm_ext_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-roberta-wwm-ext-large - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Italian DistilBERT Embeddings author: John Snow Labs name: distilbert_embeddings_BERTino date: 2022-04-12 tags: [distilbert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `BERTino` is a Italian model orginally trained by `indigo-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_BERTino_it_3.4.2_3.0_1649783809783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_BERTino_it_3.4.2_3.0_1649783809783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_BERTino| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|253.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/indigo-ai/BERTino - https://indigo.ai/en/ - https://www.corpusitaliano.it/ - https://corpora.dipintra.it/public/run.cgi/corp_info?corpname=itwac_full - https://universaldependencies.org/treebanks/it_partut/index.html - https://universaldependencies.org/treebanks/it_isdt/index.html - https://figshare.com/articles/Learning_multilingual_named_entity_recognition_from_Wikipedia/5462500 --- layout: model title: Fast Neural Machine Translation Model from Dutch to English author: John Snow Labs name: opus_mt_nl_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, nl, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `nl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_nl_en_xx_2.7.0_2.4_1609164556626.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_nl_en_xx_2.7.0_2.4_1609164556626.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_nl_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0_en_4.0.0_3.0_1657191165597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0_en_4.0.0_3.0_1657191165597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_hier_triplet_epochs_1_shard_1_kldiv_squad2.0 --- layout: model title: English DistilBertForQuestionAnswering Cased model (from aszidon) author: John Snow Labs name: distilbert_qa_custom5 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom5` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom5_en_4.3.0_3.0_1672774713572.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom5_en_4.3.0_3.0_1672774713572.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom5","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom5","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom5| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom5 --- layout: model title: Multilingual BertForQuestionAnswering model (from vanichandna) author: John Snow Labs name: bert_qa_bert_base_multilingual_cased_finetuned_squadv1 date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-squadv1` is a Multilingual model orginally trained by `vanichandna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_squadv1_xx_4.0.0_3.0_1654180211247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_squadv1_xx_4.0.0_3.0_1654180211247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_cased_finetuned_squadv1","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_cased_finetuned_squadv1","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.squad.bert.multilingual_base_cased.by_vanichandna").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_cased_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vanichandna/bert-base-multilingual-cased-finetuned-squadv1 --- layout: model title: Legal Joint Filing Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_joint_filing_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, joint_filing, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_joint_filing_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `joint-filing-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `joint-filing-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_joint_filing_agreement_bert_en_1.0.0_3.0_1669315294183.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_joint_filing_agreement_bert_en_1.0.0_3.0_1669315294183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_joint_filing_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[joint-filing-agreement]| |[other]| |[other]| |[joint-filing-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_joint_filing_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support joint-filing-agreement 1.00 0.97 0.98 31 other 0.98 1.00 0.99 65 accuracy - - 0.99 96 macro-avg 0.99 0.98 0.99 96 weighted-avg 0.99 0.99 0.99 96 ``` --- layout: model title: English BertForQuestionAnswering model (from peggyhuang) author: John Snow Labs name: bert_qa_finetune_bert_base_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `finetune-bert-base-v1` is a English model orginally trained by `peggyhuang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_finetune_bert_base_v1_en_4.0.0_3.0_1654187728270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_finetune_bert_base_v1_en_4.0.0_3.0_1654187728270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_finetune_bert_base_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_finetune_bert_base_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.base.by_peggyhuang").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_finetune_bert_base_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/peggyhuang/finetune-bert-base-v1 --- layout: model title: Recognize Entities DL Pipeline for Spanish - Small author: John Snow Labs name: entity_recognizer_sm date: 2021-03-22 tags: [open_source, spanish, entity_recognizer_sm, pipeline, es] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: es edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_es_3.0.0_3.0_1616441492784.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_es_3.0.0_3.0_1616441492784.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_sm', lang = 'es') annotations = pipeline.fullAnnotate(""Hola de John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_sm", lang = "es") val result = pipeline.fullAnnotate("Hola de John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hola de John Snow Labs! ""] result_df = nlu.load('es.ner').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:-----------------------------|:---------------------------------------|:-----------------------| | 0 | ['Hola de John Snow Labs! '] | ['Hola de John Snow Labs!'] | ['Hola', 'de', 'John', 'Snow', 'Labs!'] | [[0.1754499971866607,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'B-MISC'] | ['John Snow', 'Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|es| --- layout: model title: Legal Non Competition Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_non_competition_agreement_bert date: 2023-01-26 tags: [en, legal, classification, non_competition, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_non_competition_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `non-competition-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `non-competition-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_non_competition_agreement_bert_en_1.0.0_3.0_1674734515053.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_non_competition_agreement_bert_en_1.0.0_3.0_1674734515053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_non_competition_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[non-competition-agreement]| |[other]| |[other]| |[non-competition-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_non_competition_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support non-competition-agreement 0.88 0.93 0.90 54 other 0.96 0.94 0.95 116 accuracy - - 0.94 170 macro-avg 0.92 0.93 0.93 170 weighted-avg 0.94 0.94 0.94 170 ``` --- layout: model title: BERT multilingual base model (uncased) author: John Snow Labs name: bert_base_multilingual_uncased date: 2021-05-20 tags: [xx, multilingual, embeddings, bert, open_source] task: Embeddings language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained model on the top 102 languages with the largest Wikipedia using a masked language modeling (MLM) objective. It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in [this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference between english and English. BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it was pretrained on the raw texts only, with no humans labeling them in any way (which is why it can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it was pretrained with two objectives: - Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then runs the entire masked sentence through the model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the sentence. - Next sentence prediction (NSP): the models concatenate two masked sentences as inputs during pretraining. Sometimes they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to predict if the two sentences were following each other or not. This way, the model learns an inner representation of the English language that can then be used to extract features useful for downstream tasks: if you have a dataset of labeled sentences, for instance, you can train a standard classifier using the features produced by the BERT model as inputs. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_multilingual_uncased_xx_3.1.0_2.4_1621519949446.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_multilingual_uncased_xx_3.1.0_2.4_1621519949446.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_multilingual_uncased", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_multilingual_uncased", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("xx.embed.bert_base_multilingual_uncased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_multilingual_uncased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|xx| |Case sensitive:|true| ## Data Source [https://huggingface.co/bert-base-multilingual-uncased](https://huggingface.co/bert-base-multilingual-uncased) --- layout: model title: Legal Execution in counterparts Clause Binary Classifier author: John Snow Labs name: legclf_execution_in_counterparts_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `execution-in-counterparts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `execution-in-counterparts` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_execution_in_counterparts_clause_en_1.0.0_3.2_1660122419235.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_execution_in_counterparts_clause_en_1.0.0_3.2_1660122419235.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_execution_in_counterparts_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[execution-in-counterparts]| |[other]| |[other]| |[execution-in-counterparts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_execution_in_counterparts_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support execution-in-counterparts 1.00 0.95 0.98 42 other 0.98 1.00 0.99 128 accuracy - - 0.99 170 macro-avg 0.99 0.98 0.98 170 weighted-avg 0.99 0.99 0.99 170 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from shahma) author: John Snow Labs name: distilbert_qa_shahma_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `shahma`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_shahma_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772506673.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_shahma_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772506673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shahma_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shahma_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_shahma_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/shahma/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Applicable Law Clause Binary Classifier author: John Snow Labs name: legclf_applic_law_clause date: 2023-02-13 tags: [en, legal, classification, applicable, law, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `applic_law` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `applic_law`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_applic_law_clause_en_1.0.0_3.0_1676302179504.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_applic_law_clause_en_1.0.0_3.0_1676302179504.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_applic_law_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[applic_law]| |[other]| |[other]| |[applic_law]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_applic_law_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support applic_law 1.00 0.95 0.97 20 other 0.93 1.00 0.96 13 accuracy - - 0.97 33 macro-avg 0.96 0.97 0.97 33 weighted-avg 0.97 0.97 0.97 33 ``` --- layout: model title: Legal Liens Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_liens_bert date: 2023-03-05 tags: [en, legal, classification, clauses, liens, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Liens` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Liens`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_liens_bert_en_1.0.0_3.0_1678050683070.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_liens_bert_en_1.0.0_3.0_1678050683070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_liens_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Liens]| |[Other]| |[Other]| |[Liens]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_liens_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Liens 0.90 0.84 0.87 31 Other 0.91 0.94 0.93 53 accuracy - - 0.90 84 macro-avg 0.90 0.89 0.90 84 weighted-avg 0.90 0.90 0.90 84 ``` --- layout: model title: German asr_wav2vec2_large_xlsr_german_demo TFWav2Vec2ForCTC from marcel author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_german_demo date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_german_demo` is a German model originally trained by marcel. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_german_demo_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_german_demo_de_4.2.0_3.0_1664103879889.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_german_demo_de_4.2.0_3.0_1664103879889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_german_demo', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_german_demo", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_german_demo| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Tigrinya author: John Snow Labs name: opus_mt_en_ti date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ti, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ti` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ti_xx_2.7.0_2.4_1609170948430.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ti_xx_2.7.0_2.4_1609170948430.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ti| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal No Material Adverse Effect Clause Binary Classifier author: John Snow Labs name: legclf_no_material_adverse_effect_clause date: 2023-01-29 tags: [en, legal, classification, material, adverse, effect, clauses, no_material_adverse_effect, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-material-adverse-effect` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `no-material-adverse-effect`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_material_adverse_effect_clause_en_1.0.0_3.0_1674993807450.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_material_adverse_effect_clause_en_1.0.0_3.0_1674993807450.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_no_material_adverse_effect_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[no-material-adverse-effect]| |[other]| |[other]| |[no-material-adverse-effect]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_material_adverse_effect_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-material-adverse-effect 1.00 1.00 1.00 60 other 1.00 1.00 1.00 105 accuracy - - 1.00 165 macro-avg 1.00 1.00 1.00 165 weighted-avg 1.00 1.00 1.00 165 ``` --- layout: model title: Detect Living Species(roberta_embeddings_BR_BERTo) author: John Snow Labs name: ner_living_species_roberta date: 2022-06-22 tags: [pt, ner, clinical, licensed, roberta] task: Named Entity Recognition language: pt edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Portuguese which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `roberta_embeddings_BR_BERTo` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_roberta_pt_3.5.3_3.0_1655923058986.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_roberta_pt_3.5.3_3.0_1655923058986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = RoBertaEmbeddings.pretrained("roberta_embeddings_BR_BERTo","pt")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_roberta", "pt", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Mulher de 23 anos, de Capinota, Cochabamba, Bolívia. Ela está no nosso país há quatro anos. Frequentou o departamento de emergência obstétrica onde foi encontrada grávida de 37 semanas, com um colo dilatado de 5 cm e membranas rompidas. O obstetra de emergência realizou um teste de estreptococos negativo e solicitou um hemograma, glucose, bioquímica básica, HBV, HCV e serologia da sífilis."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = RoBertaEmbeddings.pretrained("roberta_embeddings_BR_BERTo","pt") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_roberta", "pt", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Mulher de 23 anos, de Capinota, Cochabamba, Bolívia. Ela está no nosso país há quatro anos. Frequentou o departamento de emergência obstétrica onde foi encontrada grávida de 37 semanas, com um colo dilatado de 5 cm e membranas rompidas. O obstetra de emergência realizou um teste de estreptococos negativo e solicitou um hemograma, glucose, bioquímica básica, HBV, HCV e serologia da sífilis.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("pt.med_ner.living_species.roberta").predict("""Mulher de 23 anos, de Capinota, Cochabamba, Bolívia. Ela está no nosso país há quatro anos. Frequentou o departamento de emergência obstétrica onde foi encontrada grávida de 37 semanas, com um colo dilatado de 5 cm e membranas rompidas. O obstetra de emergência realizou um teste de estreptococos negativo e solicitou um hemograma, glucose, bioquímica básica, HBV, HCV e serologia da sífilis.""") ```

## Results ```bash +-------------+-------+ |ner_chunk |label | +-------------+-------+ |Mulher |HUMAN | |grávida |HUMAN | |estreptococos|SPECIES| |HBV |SPECIES| |HCV |SPECIES| |sífilis |SPECIES| +-------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_roberta| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Size:|16.4 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.86 0.91 0.88 2827 B-SPECIES 0.52 0.86 0.65 2796 I-HUMAN 0.79 0.43 0.55 180 I-SPECIES 0.62 0.81 0.70 1099 micro-avg 0.65 0.86 0.74 6902 macro-avg 0.69 0.75 0.70 6902 weighted-avg 0.68 0.86 0.75 6902 ``` --- layout: model title: Fast Neural Machine Translation Model from Azerbaijani to English author: John Snow Labs name: opus_mt_az_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, az, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `az` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_az_en_xx_2.7.0_2.4_1609163603785.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_az_en_xx_2.7.0_2.4_1609163603785.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_az_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Japanese Bert Embeddings (Base, Character Tokenization) author: John Snow Labs name: bert_embeddings_bert_base_japanese_char date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-char` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_ja_3.4.2_3.0_1649674188981.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_ja_3.4.2_3.0_1649674188981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_char| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|334.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-char - https://github.com/google-research/bert - https://github.com/cl-tohoku/bert-japanese/tree/v1.0 - https://github.com/attardi/wikiextractor - https://taku910.github.io/mecab/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: Legal International Affairs Document Classifier (EURLEX) author: John Snow Labs name: legclf_international_affairs_bert date: 2023-03-06 tags: [en, legal, classification, clauses, international_affairs, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_international_affairs_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class International_Affairs or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `International_Affairs`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_international_affairs_bert_en_1.0.0_3.0_1678111736686.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_international_affairs_bert_en_1.0.0_3.0_1678111736686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_international_affairs_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[International_Affairs]| |[Other]| |[Other]| |[International_Affairs]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_international_affairs_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support International_Affairs 0.85 0.95 0.90 43 Other 0.96 0.86 0.91 51 accuracy - - 0.90 94 macro-avg 0.91 0.91 0.90 94 weighted-avg 0.91 0.90 0.90 94 ``` --- layout: model title: Spanish RobertaForQuestionAnswering Cased model (from stevemobs) author: John Snow Labs name: roberta_qa_quales_iberlef_squad_2 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `quales-iberlef-squad_2` is a Spanish model originally trained by `stevemobs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_quales_iberlef_squad_2_es_4.3.0_3.0_1674212017084.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_quales_iberlef_squad_2_es_4.3.0_3.0_1674212017084.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_quales_iberlef_squad_2","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_quales_iberlef_squad_2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_quales_iberlef_squad_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/stevemobs/quales-iberlef-squad_2 --- layout: model title: Sentence Entity Resolver for CPT codes (Augmented) author: John Snow Labs name: sbiobertresolve_cpt_procedures_augmented date: 2021-06-15 tags: [cpt, lincensed, en, clinical, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to CPT codes using Sentence Bert Embeddings. ## Predicted Entities CPT codes and their descriptions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_augmented_en_3.1.0_3.0_1623789734339.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_augmented_en_3.1.0_3.0_1623789734339.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") jsl_sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli','en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") cpt_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_cpt_procedures_augmented", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("cpt_code") cpt_pipelineModel = PipelineModel( stages = [ documentAssembler, jsl_sbert_embedder, cpt_resolver]) cpt_lp = LightPipeline(cpt_pipelineModel) result = cpt_lp.fullAnnotate("heart surgery") ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val cpt_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_cpt_procedures_augmented", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("cpt_code") val cpt_pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, cpt_resolver)) val cpt_lp = LightPipeline(cpt_pipelineModel) val result = cpt_lp.fullAnnotate("heart surgery") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.cpt.procedures_augmented").predict("""heart surgery""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:--------------|:----- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------|:--------------------------------------| | 0 | heart surgery | 33258 | [Cardiac surgery procedure [Operative tissue ablation and reconstruction of atria, performed at the time of other cardiac procedure(s), extensive (eg, maze procedure), without cardiopulmonary bypass (List separately in addition to code for primary procedure)], Cardiac surgery procedure [Unlisted procedure, cardiac surgery], Heart procedure [Interrogation device evaluation (in person) of intracardiac ischemia monitoring system with analysis, review, and report], Heart procedure [Insertion or removal and replacement of intracardiac ischemia monitoring system including imaging supervision and interpretation when performed and intra-operative interrogation and programming when performed; device only], ...]| [33258, 33999, 0306T, 0304T, ...] | [0.1031, 0.1031, 0.1377, 0.1377, ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_cpt_procedures_augmented| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[cpt_code]| |Language:|en| |Case sensitive:|true| ## Data Source Trained on Current Procedural Terminology dataset with `sbiobert_base_cased_mli ` sentence embeddings. --- layout: model title: Translate Brazilian Sign Language to English Pipeline author: John Snow Labs name: translate_bzs_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bzs, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bzs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bzs_en_xx_2.7.0_2.4_1609688839270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bzs_en_xx_2.7.0_2.4_1609688839270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bzs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bzs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bzs.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bzs_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Afrikaans to German author: John Snow Labs name: opus_mt_af_de date: 2021-06-01 tags: [open_source, seq2seq, translation, af, de, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: af target languages: de {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_af_de_xx_3.1.0_2.4_1622561187540.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_af_de_xx_3.1.0_2.4_1622561187540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_af_de| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Smaller BERT Sentence Embeddings (L-8_H-512_A-8) author: John Snow Labs name: sent_small_bert_L8_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L8_512_en_2.6.0_2.4_1598350686215.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L8_512_en_2.6.0_2.4_1598350686215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L8_512", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L8_512", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L8_512').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L8_512_embeddings sentence [0.07683686912059784, -0.09125291556119919, 1.... I hate cancer [0.05132533982396126, 0.16612868010997772, -0.... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L8_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1 --- layout: model title: Legal Control Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_control_agreement date: 2022-11-10 tags: [legal, licensed, classification, control, agreement, en] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_control_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `control-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `control-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_control_agreement_en_1.0.0_3.0_1668109264398.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_control_agreement_en_1.0.0_3.0_1668109264398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_control_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[control-agreement]| |[other]| |[other]| |[control-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_control_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support control-agreement 0.93 1.00 0.97 28 other 1.00 0.97 0.98 66 accuracy - - 0.98 94 macro-avg 0.97 0.98 0.98 94 weighted-avg 0.98 0.98 0.98 94 ``` --- layout: model title: Detect Assertion Status from Demographic Entities author: John Snow Labs name: assertion_oncology_demographic_binary_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects if a demographic entity refers to the patient or to someone else. ## Predicted Entities `Patient`, `Someone_Else` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_demographic_binary_wip_en_4.1.0_3.0_1664642285987.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_demographic_binary_wip_en_4.1.0_3.0_1664642285987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:----------|:------------|:-------------| | sister | Gender | Someone_Else | | age of 40 | Age | Someone_Else | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_demographic_binary_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Patient 0.94 0.94 0.94 32.0 Someone_Else 0.92 0.92 0.92 24.0 macro-avg 0.93 0.93 0.93 56.0 weighted-avg 0.93 0.93 0.93 56.0 ``` --- layout: model title: Detect 10 Different Entities in Hebrew (hebrewner_cc_300d) author: John Snow Labs name: hebrewner_cc_300d date: 2022-07-26 tags: [ner, open_source, he] task: Named Entity Recognition language: he edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses Hebrew word embeddings to find 10 different types of entities in Hebrew text. It is trained using `hebrewner_cc_300d ` word embeddings, please use the same embeddings in the pipeline. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/hebrewner_cc_300d_he_4.0.0_3.0_1658872858968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/hebrewner_cc_300d_he_4.0.0_3.0_1658872858968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python word_embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("hebrewner_cc_300d", "he") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה. אבו-ג'וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.") ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("hebrewner_cc_300d", "he") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג"וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה. אבו-ג"וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("he.ner").predict("""ח והופעל על ידי חיזבאללה. אבו-ג'וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.""") ```

## Results ```bash | | ner_chunk | entity_label | |---:|------------------:|---------------:| | 0 | 25 לאוגוסט | DATE | | 1 | השב"כ | ORG | | 2 | מוחמד אבו-ג'וייד | PERS | | 3 | ירדני | MISC_AFF | | 4 | הפת"ח | ORG | | 5 | חיזבאללה | ORG | | 6 | אבו-ג'וייד | PERS | | 7 | בגדה | LOC | | 8 | ערביי | MISC_AFF | | 9 | ישראל | LOC | | 10 | ברכבת ישראל | ORG | | 11 | בנהריה | LOC | | 12 | ישראליות | MISC_AFF | | 13 | בירדן | LOC | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|hebrewner_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|he| |Size:|14.9 MB| |Dependencies:|hebrew_cc_300d| ## References This model is trained on dataset obtained from https://www.cs.bgu.ac.il/~elhadad/nlpproj/naama/ ## Benchmarking ```bash label tp fp fn prec rec f1 I-TIME 5 2 0 0.714286 1 0.833333 I-MISC_AFF 2 0 3 1 0.4 0.571429 B-MISC_EVENT 7 0 1 1 0.875 0.933333 B-LOC 180 24 37 0.882353 0.829493 0.855107 I-ORG 124 47 38 0.725146 0.765432 0.744745 B-DATE 50 4 7 0.925926 0.877193 0.900901 I-PERS 157 10 15 0.94012 0.912791 0.926254 I-DATE 39 7 8 0.847826 0.829787 0.83871 B-MISC_AFF 132 11 9 0.923077 0.93617 0.929577 I-MISC_EVENT 6 0 2 1 0.75 0.857143 B-TIME 4 0 1 1 0.8 0.888889 I-PERCENT 8 0 0 1 1 1 I-MISC_ENT 11 3 10 0.785714 0.52381 0.628571 B-MISC_ENT 8 1 5 0.888889 0.615385 0.727273 I-LOC 79 18 23 0.814433 0.77451 0.79397 B-PERS 231 22 26 0.913044 0.898833 0.905882 B-MONEY 36 2 2 0.947368 0.947368 0.947368 B-PERCENT 28 3 0 0.903226 1 0.949152 B-ORG 166 41 37 0.801932 0.817734 0.809756 I-MONEY 61 1 1 0.983871 0.983871 0.983871 Macro-average 1334 196 225 0.899861 0.826869 0.861822 Micro-average 1334 196 225 0.871895 0.855677 0.86371 ``` --- layout: model title: Pipeline to Extraction of biomarker information author: John Snow Labs name: ner_biomarker_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_biomarker](https://nlp.johnsnowlabs.com/2021/11/26/ner_biomarker_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_biomarker_pipeline_en_3.4.1_3.0_1647871954538.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_biomarker_pipeline_en_3.4.1_3.0_1647871954538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_biomarker_pipeline", "en", "clinical/models") pipeline.annotate("Here , we report the first case of an intraductal tubulopapillary neoplasm of the pancreas with clear cell morphology . Immunohistochemistry revealed positivity for Pan-CK , CK7 , CK8/18 , MUC1 , MUC6 , carbonic anhydrase IX , CD10 , EMA , β-catenin and e-cadherin ") ``` ```scala val pipeline = new PretrainedPipeline("ner_biomarker_pipeline", "en", "clinical/models") pipeline.annotate("Here , we report the first case of an intraductal tubulopapillary neoplasm of the pancreas with clear cell morphology . Immunohistochemistry revealed positivity for Pan-CK , CK7 , CK8/18 , MUC1 , MUC6 , carbonic anhydrase IX , CD10 , EMA , β-catenin and e-cadherin ") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biomarker.pipeline").predict("""Here , we report the first case of an intraductal tubulopapillary neoplasm of the pancreas with clear cell morphology . Immunohistochemistry revealed positivity for Pan-CK , CK7 , CK8/18 , MUC1 , MUC6 , carbonic anhydrase IX , CD10 , EMA , β-catenin and e-cadherin """) ```

## Results ```bash | | ner_chunk | entity | confidence | |---:|:-------------------------|:----------------------|-------------:| | 0 | intraductal | CancerModifier | 0.9934 | | 1 | tubulopapillary | CancerModifier | 0.6403 | | 2 | neoplasm of the pancreas | CancerDx | 0.758825 | | 3 | clear cell | CancerModifier | 0.9633 | | 4 | Immunohistochemistry | Test | 0.9534 | | 5 | positivity | Biomarker_Measurement | 0.8795 | | 6 | Pan-CK | Biomarker | 0.9975 | | 7 | CK7 | Biomarker | 0.9975 | | 8 | CK8/18 | Biomarker | 0.9987 | | 9 | MUC1 | Biomarker | 0.9967 | | 10 | MUC6 | Biomarker | 0.9972 | | 11 | carbonic anhydrase IX | Biomarker | 0.937567 | | 12 | CD10 | Biomarker | 0.9974 | | 13 | EMA | Biomarker | 0.9899 | | 14 | β-catenin | Biomarker | 0.8059 | | 15 | e-cadherin | Biomarker | 0.9806 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_biomarker_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English image_classifier_vit_base_beans ViTForImageClassification from karthiksv author: John Snow Labs name: image_classifier_vit_base_beans date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_beans` is a English model originally trained by karthiksv. ## Predicted Entities `angular_leaf_spot`, `bean_rust`, `healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_en_4.1.0_3.0_1660169273658.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_en_4.1.0_3.0_1660169273658.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_beans", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_beans", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_beans| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_Tommi TFWav2Vec2ForCTC from Tommi author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_Tommi` is a Finnish model originally trained by Tommi. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi_fi_4.2.0_3.0_1664021001412.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi_fi_4.2.0_3.0_1664021001412.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_Tommi| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Word2Vec Embeddings in Urdu (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, ur, open_source] task: Embeddings language: ur edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ur_3.4.1_3.0_1647465404657.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ur_3.4.1_3.0_1647465404657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_char_extended| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|341.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/KoichiYasuoka/bert-base-japanese-char-extended --- layout: model title: English T5ForConditionalGeneration Small Cased model (from mrm8488) author: John Snow Labs name: t5_small_finetuned_imdb_sentiment date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-finetuned-imdb-sentiment` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_imdb_sentiment_en_4.3.0_3.0_1675126033560.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_imdb_sentiment_en_4.3.0_3.0_1675126033560.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_finetuned_imdb_sentiment","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_finetuned_imdb_sentiment","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_finetuned_imdb_sentiment| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|267.7 MB| ## References - https://huggingface.co/mrm8488/t5-small-finetuned-imdb-sentiment - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/pdf/1910.10683.pdf - https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67 - https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb - https://github.com/patil-suraj - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: English asr_asr_with_transformers_wav2vec2 TFWav2Vec2ForCTC from osanseviero author: John Snow Labs name: asr_asr_with_transformers_wav2vec2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_asr_with_transformers_wav2vec2` is a English model originally trained by osanseviero. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_asr_with_transformers_wav2vec2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_asr_with_transformers_wav2vec2_en_4.2.0_3.0_1664043824301.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_asr_with_transformers_wav2vec2_en_4.2.0_3.0_1664043824301.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_asr_with_transformers_wav2vec2", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_asr_with_transformers_wav2vec2", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_asr_with_transformers_wav2vec2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.6 MB| --- layout: model title: Lithuanian asr_common_voice_lithuanian_fairseq TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: asr_common_voice_lithuanian_fairseq date: 2022-09-26 tags: [wav2vec2, lt, audio, open_source, asr] task: Automatic Speech Recognition language: lt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_common_voice_lithuanian_fairseq` is a Lithuanian model originally trained by birgermoell. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_common_voice_lithuanian_fairseq_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_common_voice_lithuanian_fairseq_lt_4.2.0_3.0_1664202849902.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_common_voice_lithuanian_fairseq_lt_4.2.0_3.0_1664202849902.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_common_voice_lithuanian_fairseq", "lt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_common_voice_lithuanian_fairseq", "lt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_common_voice_lithuanian_fairseq| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lt| |Size:|228.0 MB| --- layout: model title: English RobertaForQuestionAnswering (from ydshieh) author: John Snow Labs name: roberta_qa_ydshieh_roberta_base_squad2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `ydshieh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ydshieh_roberta_base_squad2_en_4.0.0_3.0_1655735038173.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ydshieh_roberta_base_squad2_en_4.0.0_3.0_1655735038173.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ydshieh_roberta_base_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ydshieh_roberta_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_ydshieh").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ydshieh_roberta_base_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ydshieh/roberta-base-squad2 - https://www.linkedin.com/company/deepset-ai/ - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/FARM/issues/552 - https://github.com/deepset-ai/FARM - http://www.deepset.ai/jobs - https://twitter.com/deepset_ai - https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py - https://github.com/deepset-ai/haystack/discussions - https://github.com/deepset-ai/haystack/ - https://deepset.ai - https://deepset.ai/germanquad - https://deepset.ai/german-bert --- layout: model title: Catalan, Valencian asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala TFWav2Vec2ForCTC from softcatala author: John Snow Labs name: pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala date: 2022-09-24 tags: [wav2vec2, ca, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ca edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala` is a Catalan, Valencian model originally trained by softcatala. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_ca_4.2.0_3.0_1664037119962.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala_ca_4.2.0_3.0_1664037119962.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala', lang = 'ca') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala", lang = "ca") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_softcatala| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ca| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: SDOH Housing Insecurity For Classification author: John Snow Labs name: genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli date: 2023-05-30 tags: [en, licensed, biobert, sdoh, housing, generic_classifier, housing_insecurity] task: Text Classification language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting whether the patient has housing insecurity. If the clinical note includes patient housing problems, the model identifies it. If there is no housing issue or it is not mentioned in the text, it is regarded as “No_Housing_Insecurity_Or_Not_Mentioned”. The model is trained by using GenericClassifierApproach annotator. `Housing_Insecurity`: The patient has housing problems. `No_Housing_Insecurity_Or_Not_Mentioned`: The patient has no housing problems or it is not mentioned in the clinical notes. ## Predicted Entities `Housing_Insecurity`, `No_Housing_Insecurity_Or_Not_Mentioned` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_CLASSIFICATION_GENERIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli_en_4.4.2_3.0_1685474945970.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli_en_4.4.2_3.0_1685474945970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", 'en','clinical/models')\ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") features_asm = FeaturesAssembler()\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("features") generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli", 'en', 'clinical/models')\ .setInputCols(["features"])\ .setOutputCol("prediction") pipeline = Pipeline(stages=[ document_assembler, sentence_embeddings, features_asm, generic_classifier ]) text_list = ["The patient is homeless.", "Patient is a 50-year-old male who no has stable housing. He recently underwent a hip replacement surgery and has made a full recovery. ", "Patient is a 25-year-old female who has her private housing. She presented with symptoms of a urinary tract infection and was diagnosed with the condition. Her living situation has allowed her to receive prompt medical care and treatment, and she has made a full recovery. ", """Patient: Mary H. Background: Mary is a 40-year-old woman who has been diagnosed with asthma and allergies. She has been managing her conditions with medication and regular follow-up appointments with her healthcare provider. She lives in a rented apartment with her husband and two children and has been stably housed for the past five years. Presenting problem: Mary presents to the clinic for a routine check-up and reports no significant changes in her health status or symptoms related to her asthma or allergies. However, she expresses concerns about the quality of the air in her apartment and potential environmental triggers that could impact her health. Medical history: Mary has a medical history of asthma and allergies. She takes an inhaler and antihistamines to manage her conditions. Social history: Mary is married with two children and lives in a rented apartment. She and her husband both work full-time jobs and have health insurance. They have savings and are able to cover basic expenses. Assessment: The clinician assesses Mary's medical conditions and determines that her asthma and allergies are stable and well-controlled. The clinician also assesses Mary's housing situation and determines that her apartment building is in good condition and does not present any immediate environmental hazards. Plan: The clinician advises Mary to continue to monitor her health conditions and to report any changes or concerns to her healthcare team. The clinician also prescribes a referral to an allergist who can provide additional evaluation and treatment for her allergies. The clinician recommends that Mary and her family take steps to minimize potential environmental triggers in their apartment, such as avoiding smoking and using air purifiers. The clinician advises Mary to continue to maintain her stable housing situation and to seek assistance if any financial or housing issues arise. """, """Patient: Sarah L. Background: Sarah is a 35-year-old woman who has been experiencing housing insecurity for the past year. She was evicted from her apartment due to an increase in rent, which she could not afford, and has been staying with friends and family members ever since. She works as a part-time sales associate at a retail store and has no medical insurance. Presenting problem: Sarah presents to the clinic with complaints of increased stress and anxiety related to her housing insecurity. She reports feeling constantly on edge and worried about where she will sleep each night. She is also having difficulty concentrating at work and has been missing shifts due to her anxiety. Medical history: Sarah has no significant medical history and takes no medications. Social history: Sarah is currently single and has no children. She has a high school diploma but has not attended college. She has been working at her current job for three years and earns minimum wage. She has no savings and relies on her income to cover basic expenses. Assessment: The clinician assesses Sarah's mental health and determines that she is experiencing symptoms of anxiety and depression related to her housing insecurity. The clinician also assesses Sarah's housing situation and determines that she is at risk for homelessness if she is unable to secure stable housing soon. Plan: The clinician refers Sarah to a social worker who can help her connect with local housing resources, including subsidized housing programs and emergency shelters. The clinician also prescribes an antidepressant medication to help manage her symptoms of anxiety and depression. The clinician advises Sarah to continue to seek employment opportunities that may offer higher pay and stability."""] df = spark.createDataFrame(text_list, StringType()).toDF("text") result = pipeline.fit(df).transform(df) result.select("text", "prediction.result").show(truncate=100) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols("document") .setOutputCol("sentence_embeddings") val features_asm = new FeaturesAssembler() .setInputCols("sentence_embeddings") .setOutputCol("features") val generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli", "en", "clinical/models") .setInputCols("features") .setOutputCol("prediction") val pipeline = new PipelineModel().setStages(Array( document_assembler, sentence_embeddings, features_asm, generic_classifier)) val data = Seq(Array("The patient is homeless.", "Patient is a 50-year-old male who no has stable housing. He recently underwent a hip replacement surgery and has made a full recovery. ", "Patient is a 25-year-old female who has her private housing. She presented with symptoms of a urinary tract infection and was diagnosed with the condition. Her living situation has allowed her to receive prompt medical care and treatment, and she has made a full recovery. ", """Patient: Mary H. Background: Mary is a 40-year-old woman who has been diagnosed with asthma and allergies. She has been managing her conditions with medication and regular follow-up appointments with her healthcare provider. She lives in a rented apartment with her husband and two children and has been stably housed for the past five years. Presenting problem: Mary presents to the clinic for a routine check-up and reports no significant changes in her health status or symptoms related to her asthma or allergies. However, she expresses concerns about the quality of the air in her apartment and potential environmental triggers that could impact her health. Medical history: Mary has a medical history of asthma and allergies. She takes an inhaler and antihistamines to manage her conditions. Social history: Mary is married with two children and lives in a rented apartment. She and her husband both work full-time jobs and have health insurance. They have savings and are able to cover basic expenses. Assessment: The clinician assesses Mary's medical conditions and determines that her asthma and allergies are stable and well-controlled. The clinician also assesses Mary's housing situation and determines that her apartment building is in good condition and does not present any immediate environmental hazards. Plan: The clinician advises Mary to continue to monitor her health conditions and to report any changes or concerns to her healthcare team. The clinician also prescribes a referral to an allergist who can provide additional evaluation and treatment for her allergies. The clinician recommends that Mary and her family take steps to minimize potential environmental triggers in their apartment, such as avoiding smoking and using air purifiers. The clinician advises Mary to continue to maintain her stable housing situation and to seek assistance if any financial or housing issues arise. """, """Patient: Sarah L. Background: Sarah is a 35-year-old woman who has been experiencing housing insecurity for the past year. She was evicted from her apartment due to an increase in rent, which she could not afford, and has been staying with friends and family members ever since. She works as a part-time sales associate at a retail store and has no medical insurance. Presenting problem: Sarah presents to the clinic with complaints of increased stress and anxiety related to her housing insecurity. She reports feeling constantly on edge and worried about where she will sleep each night. She is also having difficulty concentrating at work and has been missing shifts due to her anxiety. Medical history: Sarah has no significant medical history and takes no medications. Social history: Sarah is currently single and has no children. She has a high school diploma but has not attended college. She has been working at her current job for three years and earns minimum wage. She has no savings and relies on her income to cover basic expenses. Assessment: The clinician assesses Sarah's mental health and determines that she is experiencing symptoms of anxiety and depression related to her housing insecurity. The clinician also assesses Sarah's housing situation and determines that she is at risk for homelessness if she is unable to secure stable housing soon. Plan: The clinician refers Sarah to a social worker who can help her connect with local housing resources, including subsidized housing programs and emergency shelters. The clinician also prescribes an antidepressant medication to help manage her symptoms of anxiety and depression. The clinician advises Sarah to continue to seek employment opportunities that may offer higher pay and stability.""")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+----------------------------------------+ | text| result| +----------------------------------------------------------------------------------------------------+----------------------------------------+ | The patient is homeless.| [Housing_Insecurity]| |Patient is a 50-year-old male who no has stable housing. He recently underwent a hip replacement ...| [Housing_Insecurity]| |Patient is a 25-year-old female who has her private housing. She presented with symptoms of a uri...|[No_Housing_Insecurity_Or_Not_Mentioned]| |Patient: Mary H. Background: Mary is a 40-year-old woman who has been diagnosed with asthma and a...|[No_Housing_Insecurity_Or_Not_Mentioned]| |Patient: Sarah L. Background: Sarah is a 35-year-old woman who has been experiencing housing ins...| [Housing_Insecurity]| +----------------------------------------------------------------------------------------------------+----------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH Project ## Benchmarking ```bash label precision recall f1-score support Housing_Insecurity 0.81 0.92 0.86 37 No_Housing_Insecurity_Or_Not_Mentioned 0.95 0.87 0.90 60 accuracy - - 0.89 97 macro-avg 0.88 0.89 0.88 97 weighted-avg 0.89 0.89 0.89 97 ``` --- layout: model title: English BertForQuestionAnswering model (from vanichandna) author: John Snow Labs name: bert_qa_muril_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `muril-finetuned-squad` is a English model orginally trained by `vanichandna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_muril_finetuned_squad_en_4.0.0_3.0_1654188629286.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_muril_finetuned_squad_en_4.0.0_3.0_1654188629286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_muril_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_muril_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_vanichandna").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_muril_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|891.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vanichandna/muril-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739600551.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739600551.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_only_classfn_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0 --- layout: model title: Translate English to Ilocano Pipeline author: John Snow Labs name: translate_en_ilo date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ilo, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ilo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ilo_xx_2.7.0_2.4_1609690588205.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ilo_xx_2.7.0_2.4_1609690588205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ilo", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ilo", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ilo').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ilo| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Castilian, Spanish BertForQuestionAnswering model (from CenIA) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-uncased-finetuned-qa-sqac` is a Castilian, Spanish model orginally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac_es_4.0.0_3.0_1654180585886.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac_es_4.0.0_3.0_1654180585886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.sqac.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/bert-base-spanish-wwm-uncased-finetuned-qa-sqac --- layout: model title: Estonain Lemmatizer author: John Snow Labs name: lemma date: 2020-11-28 task: Lemmatization language: et edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [lemmatizer, et, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_et_2.7.0_2.4_1606580379171.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_et_2.7.0_2.4_1606580379171.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenisation.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "et") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate(['üheksandana üheksas üheksanda Üheksas']) ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "et") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("üheksandana üheksas üheksanda Üheksas").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["üheksandana üheksas üheksanda Üheksas"] lemma_df = nlu.load('et.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

## Results ```bash {'lemma': [Annotation(token, 0, 10, üheksas, {'sentence': '0'}), Annotation(token, 12, 18, üheksas, {'sentence': '0'}), Annotation(token, 20, 28, üheksas, {'sentence': '0'}), Annotation(token, 30, 36, üheksas, {'sentence': '0'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|et| ## Data Source This model is trained on data obtained from [https://universaldependencies.org/](https://universaldependencies.org/) --- layout: model title: Pipeline to Detect problem, test, treatment in medical text (biobert) author: John Snow Labs name: ner_clinical_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_clinical_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_clinical_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_biobert_pipeline_en_4.3.0_3.2_1679314695992.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_biobert_pipeline_en_4.3.0_3.2_1679314695992.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_clinical_biobert_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_clinical_biobert_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------------------------------------|--------:|------:|:------------|-------------:| | 0 | congestion | 62 | 71 | PROBLEM | 0.5069 | | 1 | some mild problems with his breathing while feeding | 163 | 213 | PROBLEM | 0.694063 | | 2 | any perioral cyanosis | 233 | 253 | PROBLEM | 0.6493 | | 3 | retractions | 258 | 268 | PROBLEM | 0.9971 | | 4 | a tactile temperature | 302 | 322 | PROBLEM | 0.8294 | | 5 | Tylenol | 345 | 351 | TREATMENT | 0.665 | | 6 | some decreased p.o | 372 | 389 | PROBLEM | 0.771067 | | 7 | His normal breast-feeding | 400 | 424 | TEST | 0.736767 | | 8 | his respiratory congestion | 488 | 513 | PROBLEM | 0.745767 | | 9 | more tired | 545 | 554 | PROBLEM | 0.6514 | | 10 | fussy | 569 | 573 | PROBLEM | 0.6512 | | 11 | albuterol treatments | 637 | 656 | TREATMENT | 0.8917 | | 12 | His urine output | 675 | 690 | TEST | 0.7114 | | 13 | any diarrhea | 832 | 843 | PROBLEM | 0.73595 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding UMLS Codes author: John Snow Labs name: snomed_umls_mapping date: 2023-03-29 tags: [en, licensed, clinical, pipeline, chunk_mapping, snomed, umls] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_umls_mapper` model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapping_en_4.3.2_3.2_1680124512179.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapping_en_4.3.2_3.2_1680124512179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("snomed_umls_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(733187009 449433008 51264003) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("snomed_umls_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(733187009 449433008 51264003) ``` {:.nlu-block} ```python import nlu nlu.load("en.snomed.umls.mapping").predict("""Put your text here.""") ```

## Results ```bash | | snomed_code | umls_code | |---:|:---------------------------------|:-------------------------------| | 0 | 733187009 | 449433008 | 51264003 | C4546029 | C3164619 | C0271267 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|5.1 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English BertForQuestionAnswering Cased model (from Nadav) author: John Snow Labs name: bert_qa_macsquad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `MacSQuAD` is a English model originally trained by `Nadav`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_macsquad_en_4.0.0_3.0_1657182038098.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_macsquad_en_4.0.0_3.0_1657182038098.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_macsquad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_macsquad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_macsquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Nadav/MacSQuAD --- layout: model title: Stop Words Cleaner for Indonesian author: John Snow Labs name: stopwords_id date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: id edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, id] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_id_id_2.5.4_2.4_1594742441630.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_id_id_2.5.4_2.4_1594742441630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_id", "id") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_id", "id") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis."""] stopword_df = nlu.load('id.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=7, end=13, result='menjadi', metadata={'sentence': '0'}), Row(annotatorType='token', begin=15, end=18, result='raja', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=24, result='utara', metadata={'sentence': '0'}), Row(annotatorType='token', begin=25, end=25, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=27, end=30, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_id| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|id| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Persian DistilBERT Embeddings (from HooshvareLab) author: John Snow Labs name: distilbert_embeddings_distilbert_fa_zwnj_base date: 2022-04-12 tags: [distilbert, embeddings, fa, open_source] task: Embeddings language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-fa-zwnj-base` is a Persian model orginally trained by `HooshvareLab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_fa_zwnj_base_fa_3.4.2_3.0_1649783880670.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_fa_zwnj_base_fa_3.4.2_3.0_1649783880670.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_fa_zwnj_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fa| |Size:|282.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/HooshvareLab/distilbert-fa-zwnj-base - https://github.com/hooshvare/parsbert/issues --- layout: model title: Extract Cancer Therapies and Posology Information author: John Snow Labs name: ner_oncology_unspecific_posology_healthcare date: 2023-01-11 tags: [licensed, clinical, oncology, en, ner, treatment, posology] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of treatments and posology information using unspecific labels (low granularity). ## Predicted Entities `Posology_Information`, `Cancer_Therapy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_healthcare_en_4.2.4_3.0_1673475870938.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_healthcare_en_4.2.4_3.0_1673475870938.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:---------------------| | adriamycin | Cancer_Therapy | | 60 mg/m2 | Posology_Information | | cyclophosphamide | Cancer_Therapy | | 600 mg/m2 | Posology_Information | | over six courses | Posology_Information | | second cycle | Posology_Information | | chemotherapy | Cancer_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_unspecific_posology_healthcare| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|33.8 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Posology_Information 1435 102 210 1645 0.93 0.87 0.90 Cancer_Therapy 1281 116 125 1406 0.92 0.91 0.91 macro-avg 2716 218 335 3051 0.93 0.89 0.91 micro-avg 2716 218 335 3051 0.93 0.89 0.91 ``` --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket TFWav2Vec2ForCTC from lilitket author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket` is a English model originally trained by lilitket. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket_en_4.2.0_3.0_1664095206434.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket_en_4.2.0_3.0_1664095206434.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_lilitket| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering Cased model (from LucasS) author: John Snow Labs name: roberta_qa_robertaabsa date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robertaABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaabsa_en_4.3.0_3.0_1674222776379.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaabsa_en_4.3.0_3.0_1674222776379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertaabsa","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertaabsa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_robertaabsa| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|437.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/LucasS/robertaABSA --- layout: model title: Financial English BERT Embeddings (Number masking) author: John Snow Labs name: bert_embeddings_sec_bert_num date: 2022-04-12 tags: [bert, embeddings, en, open_source, financial] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Financial Pretrained BERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `sec-bert-num` is a English model orginally trained by `nlpaueb`. This model is the same as Bert Base but we replace every number token with a [NUM] pseudo-token handling all numeric expressions in a uniform manner, disallowing their fragmentation). If you are interested in Financial Embeddings, take a look also at these two models: [sec-base](https://nlp.johnsnowlabs.com/2022/04/12/bert_embeddings_sec_bert_base_en_3_0.html): Same as Bert Base but trained with financial documents. [sec-shape](https://nlp.johnsnowlabs.com/2022/04/12/bert_embeddings_sec_bert_sh_en_3_0.html): Same as Bert sec-base but we replace numbers with pseudo-tokens that represent the number’s shape, so numeric expressions (of known shapes) are no longer fragmented, e.g., '53.2' becomes '[XX.X]' and '40,200.5' becomes '[XX,XXX.X]'. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_sec_bert_num_en_3.4.2_3.0_1649759295271.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_sec_bert_num_en_3.4.2_3.0_1649759295271.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_sec_bert_num| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/nlpaueb/sec-bert-num - https://arxiv.org/abs/2203.06482 - http://nlp.cs.aueb.gr/ --- layout: model title: English AlbertForQuestionAnswering Large model (from elgeish) author: John Snow Labs name: albert_qa_cs224n_squad2.0_large_v2 date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cs224n-squad2.0-albert-large-v2` is a English model originally trained by `elgeish`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_cs224n_squad2.0_large_v2_en_4.0.0_3.0_1656064295571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_cs224n_squad2.0_large_v2_en_4.0.0_3.0_1656064295571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_cs224n_squad2.0_large_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_cs224n_squad2.0_large_v2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.albert.large_v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_cs224n_squad2.0_large_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|63.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2 - http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf - https://rajpurkar.github.io/SQuAD-explorer/ - https://github.com/elgeish/squad/tree/master/data --- layout: model title: Turkish DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_tr_cased date: 2022-04-12 tags: [distilbert, embeddings, tr, open_source] task: Embeddings language: tr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-tr-cased` is a Turkish model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_tr_cased_tr_3.4.2_3.0_1649783637258.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_tr_cased_tr_3.4.2_3.0_1649783637258.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_tr_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|tr| |Size:|216.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-tr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Cyberbullying Classifier author: John Snow Labs name: classifierdl_use_cyberbullying date: 2021-01-09 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, en, classifier] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Identify Racism, Sexism or Neutral tweets. ## Predicted Entities `neutral`, `racism`, `sexism` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_CYBERBULLYING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN_CYBERBULLYING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_cyberbullying_en_2.7.1_2.4_1610188083627.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_cyberbullying_en_2.7.1_2.4_1610188083627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------------------------------------------------+------------+ |document |class | +--------------------------------------------------------------------------------------------------------+------------+ |@geeky_zekey Thanks for showing again that blacks are the biggest racists. Blocked. | racism | +--------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_cyberbullying| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|tfhub_use| ## Data Source This model is trained on cyberbullying detection dataset. https://raw.githubusercontent.com/dhavalpotdar/cyberbullying-detection/master/data/data/data.csv ## Benchmarking ```bash precision recall f1-score support neutral 0.72 0.76 0.74 700 racism 0.89 0.94 0.92 773 sexism 0.82 0.71 0.76 622 accuracy 0.81 2095 macro avg 0.81 0.80 0.80 2095 weighted avg 0.81 0.81 0.81 2095 ``` --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from haesun) author: John Snow Labs name: xlmroberta_ner_haesun_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `haesun`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_haesun_base_finetuned_panx_de_4.1.0_3.0_1660433456248.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_haesun_base_finetuned_panx_de_4.1.0_3.0_1660433456248.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_haesun_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_haesun_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_haesun_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/haesun/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English BertForQuestionAnswering Cased model (from aozorahime) author: John Snow Labs name: bert_qa_my_new_model date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `my-new-model` is a English model originally trained by `aozorahime`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_my_new_model_en_4.0.0_3.0_1657190466916.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_my_new_model_en_4.0.0_3.0_1657190466916.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_my_new_model","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_my_new_model","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_my_new_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aozorahime/my-new-model --- layout: model title: Pipeline to Detect diseases in medical text (biobert) author: John Snow Labs name: ner_diseases_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_diseases_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_diseases_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_pipeline_en_4.3.0_3.2_1679315318481.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_pipeline_en_4.3.0_3.2_1679315318481.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_diseases_biobert_pipeline", "en", "clinical/models") text = '''Indomethacin resulted in histopathologic findings typical of interstitial cystitis, such as leaky bladder epithelium and mucosal mastocytosis. The true incidence of nonsteroidal anti-inflammatory drug-induced cystitis in humans must be clarified by prospective clinical trials. An open-label phase II study of low-dose thalidomide in androgen-independent prostate cancer.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_diseases_biobert_pipeline", "en", "clinical/models") val text = "Indomethacin resulted in histopathologic findings typical of interstitial cystitis, such as leaky bladder epithelium and mucosal mastocytosis. The true incidence of nonsteroidal anti-inflammatory drug-induced cystitis in humans must be clarified by prospective clinical trials. An open-label phase II study of low-dose thalidomide in androgen-independent prostate cancer." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.diseases_biobert.pipeline").predict("""Indomethacin resulted in histopathologic findings typical of interstitial cystitis, such as leaky bladder epithelium and mucosal mastocytosis. The true incidence of nonsteroidal anti-inflammatory drug-induced cystitis in humans must be clarified by prospective clinical trials. An open-label phase II study of low-dose thalidomide in androgen-independent prostate cancer.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:------------|-------------:| | 0 | interstitial cystitis | 61 | 81 | Disease | 0.99655 | | 1 | mastocytosis | 129 | 140 | Disease | 0.8569 | | 2 | cystitis | 209 | 216 | Disease | 0.9717 | | 3 | prostate cancer | 355 | 369 | Disease | 0.85965 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_diseases_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_rbt3 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rbt3` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt3_zh_4.2.4_3.0_1670327065192.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt3_zh_4.2.4_3.0_1670327065192.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt3","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbt3","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt3| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|144.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt3 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English asr_wav2vec2_base_test TFWav2Vec2ForCTC from cahya author: John Snow Labs name: pipeline_asr_wav2vec2_base_test date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_test` is a English model originally trained by cahya. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_test_en_4.2.0_3.0_1664035679567.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_test_en_4.2.0_3.0_1664035679567.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_test', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_test", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|348.7 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from jgammack) author: John Snow Labs name: distilbert_qa_mtl_base_uncased_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `MTL-distilbert-base-uncased-squad` is a English model originally trained by `jgammack`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_mtl_base_uncased_squad_en_4.3.0_3.0_1672765509575.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_mtl_base_uncased_squad_en_4.3.0_3.0_1672765509575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mtl_base_uncased_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mtl_base_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_mtl_base_uncased_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/jgammack/MTL-distilbert-base-uncased-squad --- layout: model title: Russian RoBERTa Embeddings author: John Snow Labs name: roberta_embeddings_ruRoberta_large date: 2022-04-14 tags: [roberta, embeddings, ru, open_source] task: Embeddings language: ru edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `ruRoberta-large` is a Russian model orginally trained by `sberbank-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_ruRoberta_large_ru_3.4.2_3.0_1649947722752.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_ruRoberta_large_ru_3.4.2_3.0_1649947722752.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_ruRoberta_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ru| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/sberbank-ai/ruRoberta-large - https://sberdevices.ru/ --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from leabum) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_cuad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-cuad` is a English model originally trained by `leabum`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_cuad_en_4.3.0_3.0_1672767856883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_cuad_en_4.3.0_3.0_1672767856883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_cuad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_cuad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_cuad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/leabum/distilbert-base-uncased-finetuned-cuad --- layout: model title: Emotional Stress Classifier (BERT) author: John Snow Labs name: bert_sequence_classifier_stress date: 2022-06-28 tags: [sequence_classification, bert, en, licensed, stress, mental, public_health] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [PHS-BERT-based](https://huggingface.co/publichealthsurveillance/PHS-BERT) classifier that can classify whether the content of a text expresses emotional stress. ## Predicted Entities `no stress`, `stress` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_STRESS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_stress_en_4.0.0_3.0_1656438010655.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_stress_en_4.0.0_3.0_1656438010655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rpv_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rpv_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_rpv_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/rpv/distilbert-base-uncased-finetuned-squad --- layout: model title: Japanese asr_wav2vec2_large_xlsr_japanese_hiragana TFWav2Vec2ForCTC from vumichien author: John Snow Labs name: asr_wav2vec2_large_xlsr_japanese_hiragana date: 2022-09-25 tags: [wav2vec2, ja, audio, open_source, asr] task: Automatic Speech Recognition language: ja edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_japanese_hiragana` is a Japanese model originally trained by vumichien. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_japanese_hiragana_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_japanese_hiragana_ja_4.2.0_3.0_1664122415445.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_japanese_hiragana_ja_4.2.0_3.0_1664122415445.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_japanese_hiragana", "ja")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_japanese_hiragana", "ja") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_japanese_hiragana| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ja| |Size:|1.2 GB| --- layout: model title: Pipeline to Detect PHI in Text (enriched-biobert) author: John Snow Labs name: ner_deid_enriched_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, enriched_biobert, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_enriched_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_enriched_biobert_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_pipeline_en_3.4.1_3.0_1647868393082.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_biobert_pipeline_en_3.4.1_3.0_1647868393082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_enriched_biobert_pipeline", "en", "clinical/models") pipeline.annotate("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_enriched_biobert_pipeline", "en", "clinical/models") pipeline.annotate("A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.ner_enriched_biobert.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash +-----------------------------+------------+ |chunks |entities | +-----------------------------+------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson, Ora |DOCTOR | |7194334 |PHONE | |01/13/93 |DATE | |Oliveira |DOCTOR | |1-11-2000 |DATE | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street |STREET | |(302) 786-5227 |PHONE | |Brothers Coal-Mine |ORGANIZATION| +-----------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_enriched_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_el2 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-el2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_el2_en_4.3.0_3.0_1675123363126.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_el2_en_4.3.0_3.0_1675123363126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_el2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_el2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_el2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|59.2 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-el2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_dl8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-dl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl8_en_4.3.0_3.0_1675118817239.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl8_en_4.3.0_3.0_1675118817239.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_dl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_dl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_dl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|164.0 MB| ## References - https://huggingface.co/google/t5-efficient-small-dl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_el2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-el2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el2_en_4.3.0_3.0_1675111126562.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el2_en_4.3.0_3.0_1675111126562.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_el2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_el2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_el2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|339.1 MB| ## References - https://huggingface.co/google/t5-efficient-base-el2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Detect PHI for Deidentification (Glove - Subentity) author: John Snow Labs name: ner_deid_subentity_glove date: 2021-06-06 tags: [ner, deid, licensed, en, glove, clinical] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Absolute) is a Named Entity Recognition model that annotates text to find protected health information that may need to be deidentified. It detects 23 entities. This ner model is trained with combination of i2b2 train set and augmented version of i2b2 train set using Glove-100d embeddings. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `MEDICALRECORD`, `ORGANIZATION`, `DOCTOR`, `USERNAME`, `PROFESSION`, `HEALTHPLAN`, `URL`, `CITY`, `DATE`, `LOCATION-OTHER`, `STATE`, `PATIENT`, `DEVICE`, `COUNTRY`, `ZIP`, `PHONE`, `HOSPITAL`, `EMAIL`, `IDNUM`, `SREET`, `BIOID`, `FAX`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/deid_ner_subentity_glove_en_3.0.4_3.0_1623015533538.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/deid_ner_subentity_glove_en_3.0.4_3.0_1623015533538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") glove_embeddings = WordEmbeddingsModel.pretrained('glove_100d') \ .setInputCols(['sentence', 'token']) \ .setOutputCol('embeddings') deid_ner = MedicalNerModel.pretrained("ner_deid_subentity_glove", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk_subentity") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, glove_embeddings, deid_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame(pd.DataFrame({"text": ["""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227."""]}))) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val glove_embeddings = WordEmbeddingsModel.pretrained("glove_100d") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val deid_ner = MedicalNerModel.pretrained("ner_deid_subentity_glove", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk_subentity") val nlpPipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, glove_embeddings, deid_ner, ner_converter)) val result = nlpPipeline.fit(Seq.empty["A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227."].toDS.toDF("text")).transform(data) ```

## Results ```bash +-----------------------------+-------------+ |chunk |ner_label | +-----------------------------+-------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson, Ora |PATIENT | |7194334 |MEDICALRECORD| |01/13/93 |DATE | |Oliveira |DOCTOR | |25 |AGE | |1-11-2000 |DATE | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street |STREET | |+1 (302) 786-5227 |PHONE | +-----------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_glove| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source A custom data set which is created from the i2b2-PHI train and the augmented version of the i2b2-PHI train set is used. --- layout: model title: Translate Yoruba to English Pipeline author: John Snow Labs name: translate_yo_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, yo, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `yo` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_yo_en_xx_2.7.0_2.4_1609688368244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_yo_en_xx_2.7.0_2.4_1609688368244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_yo_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_yo_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.yo.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_yo_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes (Disease or Syndrome) author: John Snow Labs name: sbiobertresolve_umls_disease_syndrome date: 2021-10-11 tags: [entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.2.3 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities to UMLS CUI codes. It is trained on `2021AB` UMLS dataset. The complete dataset has 127 different categories, and this model is trained on the `Disease or Syndrome` category using `sbiobert_base_cased_mli` embeddings. ## Predicted Entities `Predicts UMLS codes for Diseases & Syndromes medical concepts` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_disease_syndrome_en_3.2.3_3.0_1633911418710.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_disease_syndrome_en_3.2.3_3.0_1633911418710.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_umls_disease_syndrome``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Cerebrovascular_Disease, Communicable_Disease, Diabetes,Disease_Syndrome_Disorder, Heart_Disease, Hyperlipidemia, Hypertension,Injury_or_Poisoning, Kidney_Disease, Obesity, Oncological, Overweight, Psychological_Condition, Symptom, VS_Finding, ImagingFindings, EKG_Findings``` set in ```.setWhiteList()```.

## Results ```bash | | chunk | code | code_description | all_k_code_desc | all_k_codes | |---:|:--------------------------------------|:---------|:--------------------------------------|:-------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | gestational diabetes mellitus | C0085207 | gestational diabetes mellitus | ['C0085207', 'C0032969', 'C2063017', 'C1283034', 'C0271663'] | ['gestational diabetes mellitus', 'pregnancy diabetes mellitus', 'pregnancy complicated by diabetes mellitus', 'maternal diabetes mellitus', 'gestational diabetes mellitus, a2'] | | 1 | subsequent type two diabetes mellitus | C0348921 | pre-existing type 2 diabetes mellitus | ['C0348921', 'C1719939', 'C0011860', 'C0877302', 'C0271640'] | ['pre-existing type 2 diabetes mellitus', 'disorder associated with type 2 diabetes mellitus', 'diabetes mellitus, type 2', 'insulin-requiring type 2 diabetes mellitus', 'secondary diabetes mellitus'] | | 2 | HTG-induced pancreatitis | C0376670 | alcohol-induced pancreatitis | ['C0376670', 'C1868971', 'C4302243', 'C0267940', 'C2350449'] | ['alcohol-induced pancreatitis', 'toxic pancreatitis', 'igg4-related pancreatitis', 'hemorrhage pancreatitis', 'graft pancreatitis'] | | 3 | an acute hepatitis | C0019159 | acute hepatitis | ['C0019159', 'C0276434', 'C0267797', 'C1386146', 'C2063407'] | ['acute hepatitis a', 'acute hepatitis a', 'acute hepatitis', 'acute infectious hepatitis', 'acute hepatitis e'] | | 4 | obesity | C0028754 | obesity | ['C0028754', 'C0342940', 'C0342942', 'C0857116', 'C1561826'] | ['obesity', 'abdominal obesity', 'generalized obesity', 'obesity gross', 'overweight and obesity'] | | 5 | polyuria | C0018965 | hematuria | ['C0018965', 'C0151582', 'C3888890', 'C0268556', 'C2936921'] | ['hematuria', 'uricosuria', 'polyuria-polydipsia syndrome', 'saccharopinuria', 'saccharopinuria'] | | 6 | polydipsia | C0268813 | primary polydipsia | ['C0268813', 'C0030508', 'C3888890', 'C0393777', 'C0206085'] | ['primary polydipsia', 'parasomnia', 'polyuria-polydipsia syndrome', 'hypnogenic paroxysmal dystonias', 'periodic hypersomnias'] | | 7 | poor appetite | C0003123 | lack of appetite | ['C0003123', 'C0011168', 'C0162429', 'C1282895', 'C0039338'] | ['lack of appetite', 'poor swallowing', 'poor nutrition', 'neurologic unpleasant taste', 'taste dis'] | | 8 | vomiting | C0152164 | periodic vomiting | ['C0152164', 'C0267172', 'C0152517', 'C0011119', 'C0152227'] | ['periodic vomiting', 'habit vomiting', 'viral vomiting', 'choking', 'tearing'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_umls_disease_syndrome| |Compatibility:|Healthcare NLP 3.2.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[output]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on `2021AB` UMLS dataset's ' Disease or Syndrome` category. https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: English image_classifier_vit_rare_puppers ViTForImageClassification from eliwill author: John Snow Labs name: image_classifier_vit_rare_puppers date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rare_puppers` is a English model originally trained by eliwill. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_puppers_en_4.1.0_3.0_1660167215470.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_puppers_en_4.1.0_3.0_1660167215470.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rare_puppers", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rare_puppers", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rare_puppers| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Translate English to Kinyarwanda Pipeline author: John Snow Labs name: translate_en_rw date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, rw, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `rw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_rw_xx_2.7.0_2.4_1609686452332.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_rw_xx_2.7.0_2.4_1609686452332.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_rw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_rw", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.rw').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_rw| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from South Caucasian Languages to English author: John Snow Labs name: opus_mt_ccs_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ccs, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ccs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ccs_en_xx_2.7.0_2.4_1609169090699.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ccs_en_xx_2.7.0_2.4_1609169090699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ccs_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English Bert Embeddings (from anferico) author: John Snow Labs name: bert_embeddings_bert_for_patents date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-for-patents` is a English model orginally trained by `anferico`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_for_patents_en_3.4.2_3.0_1649671629607.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_for_patents_en_3.4.2_3.0_1649671629607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_for_patents| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/anferico/bert-for-patents - https://cloud.google.com/blog/products/ai-machine-learning/how-ai-improves-patent-analysis - https://services.google.com/fh/files/blogs/bert_for_patents_white_paper.pdf - https://github.com/google/patents-public-data/blob/master/models/BERT%20for%20Patents.md - https://github.com/ec-jrc/Patents4IPPC - https://picampus-school.com/ - https://ec.europa.eu/jrc/en --- layout: model title: Spanish RoBERTa Embeddings (Large) author: John Snow Labs name: roberta_embeddings_roberta_large_bne date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-large-bne` is a Spanish model orginally trained by `PlanTL-GOB-ES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_large_bne_es_3.4.2_3.0_1649945069671.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_large_bne_es_3.4.2_3.0_1649945069671.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_large_bne| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|848.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-large-bne - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - http://www.bne.es/en/Inicio/index.html - https://arxiv.org/abs/1907.11692 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Liability and Contra-Liability NER (Small) author: John Snow Labs name: finner_contraliability date: 2022-12-15 tags: [en, finance, contra, liability, licensed, ner] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a financial model to detect LIABILITY and CONTRA_LIABILITY mentions in texts. - CONTRA_LIABILITY: Negative liability account that offsets the liability account (e.g. paying a debt) - LIABILITY: Current or Long-Term Liability (not from stockholders) Please note this model requires some tokenization configuration to extract the currency (see python snippet below). ## Predicted Entities `LIABILITY`, `CONTRA_LIABILITY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_contraliability_en_1.0.0_3.0_1671136444267.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_contraliability_en_1.0.0_3.0_1671136444267.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_contraliability", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Reducing total debt continues to be a top priority , and we remain on track with our target of reducing overall debt levels by $ 15 billion by the end of 2025 ."""]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("text"), F.expr("cols['1']['entity']").alias("label")).show(200, truncate = False) ```

## Results ```bash +---------+------------------+----------+ | token| ner_label|confidence| +---------+------------------+----------+ | Reducing| O| 0.9997| | total| B-LIABILITY| 0.7884| | debt| I-LIABILITY| 0.8479| |continues| O| 1.0| | to| O| 1.0| | be| O| 1.0| | a| O| 1.0| | top| O| 1.0| | priority| O| 1.0| | ,| O| 1.0| | and| O| 1.0| | we| O| 1.0| | remain| O| 1.0| | on| O| 1.0| | track| O| 1.0| | with| O| 1.0| | our| O| 1.0| | target| O| 1.0| | of| O| 1.0| | reducing| O| 0.9993| | overall| O| 0.9969| | debt|B-CONTRA_LIABILITY| 0.5686| | levels|I-CONTRA_LIABILITY| 0.6611| | by| O| 0.9996| | $| O| 1.0| | 15| O| 1.0| | billion| O| 1.0| | by| O| 1.0| | the| O| 1.0| | end| O| 1.0| | of| O| 1.0| | 2025| O| 1.0| | .| O| 1.0| +---------+------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_contraliability| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References In-house annotations on Earning Calls and 10-K Filings combined. ## Benchmarking ```bash label precision recall f1-score support B-CONTRA_LIABILITY 0.7660 0.7200 0.7423 50 B-LIABILITY 0.8947 0.8990 0.8969 208 I-CONTRA_LIABILITY 0.7838 0.6304 0.6988 46 I-LIABILITY 0.8780 0.8929 0.8854 411 accuracy - - 0.9805 8299 macro-avg 0.8626 0.8267 0.8429 8299 weighted-avg 0.9803 0.9805 0.9803 8299 ``` --- layout: model title: Recognize Entities DL Pipeline for Norwegian (Bokmal) - Medium author: John Snow Labs name: entity_recognizer_md date: 2021-03-22 tags: [open_source, norwegian_bokmal, entity_recognizer_md, pipeline, "no"] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: "no" edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_no_3.0.0_3.0_1616451623734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_no_3.0.0_3.0_1616451623734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_md', lang = 'no') annotations = pipeline.fullAnnotate(""Hei fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_md", lang = "no") val result = pipeline.fullAnnotate("Hei fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei fra John Snow Labs! ""] result_df = nlu.load('no.ner.md').predict(text) result_df ```

## Results ```bash +-----------------------------------------------------------------------------------------------------------------------+------------------+ |text |result | +-----------------------------------------------------------------------------------------------------------------------+------------------+ |hi does anybody have feet aches with anxiety, i do suffer from anxiety but never had anything wrong with my feet before|[Other] | |My son has been to two doctors who gave him antibiotic drops but they also say the problem might related to allergies. |[Consulted_By_HCP]| +-----------------------------------------------------------------------------------------------------------------------+------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_hcp_consult| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset “Hello,I’m 20 year old girl. I’m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I’m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I’m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I’m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you.” ## Benchmarking ```bash label precision recall f1-score support Consulted_By_HCP 0.670412 0.730612 0.699219 245 Other 0.848624 0.807860 0.827740 458 accuracy - - 0.780939 703 macro_avg 0.759518 0.769236 0.763480 703 weighted_avg 0.786516 0.780939 0.782950 703 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Ewe author: John Snow Labs name: opus_mt_en_ee date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ee, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ee` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ee_xx_2.7.0_2.4_1609167230649.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ee_xx_2.7.0_2.4_1609167230649.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ee| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Termination Clause Binary Classifier (md) author: John Snow Labs name: legclf_termination_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `termination` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `termination` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_termination_md_en_1.0.0_3.0_1669376522291.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_termination_md_en_1.0.0_3.0_1669376522291.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_termination_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[termination]| |[other]| |[other]| |[termination]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_termination_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support other 0.95 0.97 0.96 39 termination 0.97 0.94 0.95 32 accuracy 0.96 71 macro avg 0.96 0.96 0.96 71 weighted avg 0.96 0.96 0.96 71 ``` --- layout: model title: Chinese Bert Embeddings (from celtics1863) author: John Snow Labs name: bert_embeddings_env_bert_chinese date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `env-bert-chinese` is a Chinese model orginally trained by `celtics1863`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_env_bert_chinese_zh_3.4.2_3.0_1649670664238.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_env_bert_chinese_zh_3.4.2_3.0_1649670664238.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_env_bert_chinese| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|384.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/celtics1863/env-bert-chinese --- layout: model title: NER Pipeline for 10 African Languages author: John Snow Labs name: xlm_roberta_large_token_classifier_masakhaner_pipeline date: 2022-06-27 tags: [masakhaner, african, xlm_roberta, multilingual, pipeline, amharic, hausa, igbo, kinyarwanda, luganda, swahilu, wolof, yoruba, nigerian, pidgin, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on [xlm_roberta_large_token_classifier_masakhaner](https://nlp.johnsnowlabs.com/2021/12/06/xlm_roberta_large_token_classifier_masakhaner_xx.html) ner model which is imported from `HuggingFace`. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/Ner_masakhaner/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Ner_masakhaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_masakhaner_pipeline_xx_4.0.0_3.0_1656369154380.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_masakhaner_pipeline_xx_4.0.0_3.0_1656369154380.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python masakhaner_pipeline = PretrainedPipeline("xlm_roberta_large_token_classifier_masakhaner_pipeline", lang = "xx") masakhaner_pipeline.annotate("አህመድ ቫንዳ ከ3-10-2000 ጀምሮ በአዲስ አበባ ኖሯል።") ``` ```scala val masakhaner_pipeline = new PretrainedPipeline("xlm_roberta_large_token_classifier_masakhaner_pipeline", lang = "xx") val masakhaner_pipeline.annotate("አህመድ ቫንዳ ከ3-10-2000 ጀምሮ በአዲስ አበባ ኖሯል።") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hiiii23_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hiiii23_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_hiiii23").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hiiii23_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hiiii23/distilbert-base-uncased-finetuned-squad --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_512 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_512_zh_4.2.4_3.0_1670325937593.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_512_zh_4.2.4_3.0_1670325937593.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_512| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|90.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-512 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Spanish Named Entity Recognition (Base, CAPITEL competition at IberLEF 2020 dataset) author: John Snow Labs name: roberta_ner_roberta_base_bne_capitel_ner date: 2022-05-03 tags: [roberta, ner, open_source, es] task: Named Entity Recognition language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-base-bne-capitel-ner` is a Spanish model orginally trained by `PlanTL-GOB-ES`. ## Predicted Entities `ORG`, `LOC`, `PER`, `OTH` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_base_bne_capitel_ner_es_3.4.2_3.0_1651593219771.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_base_bne_capitel_ner_es_3.4.2_3.0_1651593219771.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_roberta_base_bne_capitel_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|457.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne-capitel-ner - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://sites.google.com/view/capitel2020 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Detect PHI for Deidentification purposes (Portuguese) author: John Snow Labs name: ner_deid_subentity date: 2022-04-13 tags: [deid, deidentification, pt, licensed] task: De-identification language: pt edition: Healthcare NLP 3.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Portuguese) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 15 entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `ID`, `STREET`, `SEX`, `EMAIL`, `ZIP`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pt_3.4.2_3.0_1649840643338.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pt_3.4.2_3.0_1649840643338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.WordEmbeddingsModel.pretrained("w2v_cc_300d", "pt")\ .setInputCols(["sentence","token"])\ .setOutputCol("word_embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_subentity", "pt", "clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner, ner_converter]) text = [''' Detalhes do paciente. Nome do paciente: Pedro Gonçalves NHC: 2569870. Endereço: Rua Das Flores 23. Cidade/ Província: Porto. Código Postal: 21754-987. Dados de cuidados. Data de nascimento: 10/10/1963. Idade: 53 anos Sexo: Homen Data de admissão: 17/06/2016. Doutora: Maria Santos '''] data = spark.createDataFrame([text]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "pt") .setInputCols(Array("sentence","token")) .setOutputCol("word_embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_subentity", "pt", "clinical/models") .setInputCols(Array("sentence","token","word_embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner, ner_converter)) val text = """Detalhes do paciente. Nome do paciente: Pedro Gonçalves NHC: 2569870. Endereço: Rua Das Flores 23. Cidade/ Província: Porto. Código Postal: 21754-987. Dados de cuidados. Data de nascimento: 10/10/1963. Idade: 53 anos Sexo: Homen Data de admissão: 17/06/2016. Doutora: Maria Santos""" val data = Seq(text).toDF("text") val results = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("pt.med_ner.deid.subentity").predict(""" Detalhes do paciente. Nome do paciente: Pedro Gonçalves NHC: 2569870. Endereço: Rua Das Flores 23. Cidade/ Província: Porto. Código Postal: 21754-987. Dados de cuidados. Data de nascimento: 10/10/1963. Idade: 53 anos Sexo: Homen Data de admissão: 17/06/2016. Doutora: Maria Santos """) ```

## Results ```bash +-----------------+------------+ |chunk |ner_label | +-----------------+------------+ |Pedro Gonçalves |PATIENT | |2569870 |ID | |Rua Das Flores 23|STREET | |Porto |ORGANIZATION| |21754-987 |ID | |10/10/1963 |DATE | |53 |AGE | |17/06/2016 |DATE | |Maria Santos |DOCTOR | +-----------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Size:|15.0 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 2142.0 186.0 59.0 2201.0 0.9201 0.9732 0.9459 HOSPITAL 248.0 10.0 46.0 294.0 0.9612 0.8435 0.8986 DATE 1306.0 26.0 15.0 1321.0 0.9805 0.9886 0.9845 ORGANIZATION 3038.0 31.0 156.0 3194.0 0.9899 0.9512 0.9701 CITY 1836.0 58.0 15.0 1851.0 0.9694 0.9919 0.9805 ID 56.0 8.0 7.0 63.0 0.875 0.8889 0.8819 STREET 155.0 0.0 0.0 155.0 1.0 1.0 1.0 SEX 658.0 20.0 19.0 677.0 0.9705 0.9719 0.9712 EMAIL 131.0 0.0 0.0 131.0 1.0 1.0 1.0 ZIP 125.0 2.0 0.0 125.0 0.9843 1.0 0.9921 PROFESSION 237.0 15.0 39.0 276.0 0.9405 0.8587 0.8977 PHONE 64.0 2.0 0.0 64.0 0.9697 1.0 0.9846 COUNTRY 502.0 27.0 74.0 576.0 0.949 0.8715 0.9086 DOCTOR 461.0 35.0 31.0 492.0 0.9294 0.937 0.9332 AGE 538.0 17.0 8.0 546.0 0.9694 0.9853 0.9773 macro - - - - - - 0.9551 micro - - - - - - 0.9619 ``` --- layout: model title: Word2Vec Embeddings in Georgian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, ka, open_source] task: Embeddings language: ka edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ka_3.4.1_3.0_1647374663429.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ka_3.4.1_3.0_1647374663429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|ka| |Size:|909.1 MB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Table recognition author: John Snow Labs name: table_recognition date: 2023-01-03 tags: [en, licensed, ocr, table_recognition] task: Table Recognition language: en nav_key: models edition: Visual NLP 4.1.0 spark_version: 3.3.0 supported: true annotator: TableRecognition article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model shows the capabilities for table recognition and free-text extraction using OCR techniques. For table recognition is proposed a CascadTabNet model. CascadeTabNet is a machine learning model for table detection in document images. It is based on a cascaded architecture, which is a two-stage process where the model first detects candidate regions that may contain tables, and then classifies these regions as tables or non-tables. The model is trained using a dataset of document images, where the tables have been manually annotated. The benchmark results show that the model is able to detect tables in document images with high accuracy. On the ICDAR2013 table competition dataset, CascadeTabNet achieved an F1-score of 0.85, which is considered a good score in this dataset. On the COCO-Text dataset, the model achieved a precision of 0.82 and a recall of 0.79, which are also considered good scores. In addition, the model has been evaluated on the UNLV dataset, where it achieved a precision of 0.87 and a recall of 0.83. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/IMAGE_TABLE_DETECTION/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/blob/master/tutorials/Certification_Trainings/2.2.Spark_OCR_training_Table_recognition.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) # Detect tables on the page using pretrained model # It can be finetuned for have more accurate results for more specific documents table_detector = ImageTableDetector.pretrained("general_model_table_detection_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("region") # Draw detected region's with table to the page draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("region") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.red) # Extract table regions to separate images splitter = ImageSplitRegions() \ .setInputCol("image") \ .setInputRegionsCol("region") \ .setOutputCol("table_image") \ .setDropCols("image") # Detect cells on the table image cell_detector = ImageTableCellDetector() \ .setInputCol("table_image") \ .setOutputCol("cells") \ .setAlgoType("morphops") \ .setDrawDetectedLines(True) # Extract text from the detected cells table_recognition = ImageCellsToTextTable() \ .setInputCol("table_image") \ .setCellsCol('cells') \ .setMargin(3) \ .setStrip(True) \ .setOutputCol('table') # Erase detected table regions fill_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("region") \ .setOutputCol("image_1") \ .setRectColor(Color.white) \ .setFilledRect(True) # OCR ocr = ImageToText() \ .setInputCol("image_1") \ .setOutputCol("text") \ .setOcrParams(["preserve_interword_spaces=1", ]) \ .setKeepLayout(True) \ .setOutputSpaceCharacterWidth(8) pipeline_table = PipelineModel(stages=[ binary_to_image, table_detector, draw_regions, fill_regions, splitter, cell_detector, table_recognition, ocr ]) imagePath = "/content/cTDaR_t10096.jpg" df = spark.read.format("binaryFile").load(imagePath) tables_results = pipeline_table.transform(df).cache() ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) # Detect tables on the page using pretrained model # It can be finetuned for have more accurate results for more specific documents val table_detector = ImageTableDetector .pretrained("general_model_table_detection_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("region") # Draw detected region's with table to the page val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("region") .setOutputCol("image_with_regions") .setRectColor(Color.red) # Extract table regions to separate images val splitter = new ImageSplitRegions() .setInputCol("image") .setInputRegionsCol("region") .setOutputCol("table_image") .setDropCols("image") # Detect cells on the table image val cell_detector = new ImageTableCellDetector() .setInputCol("table_image") .setOutputCol("cells") .setAlgoType("morphops") .setDrawDetectedLines(True) # Extract text from the detected cells val table_recognition = new ImageCellsToTextTable() .setInputCol("table_image") .setCellsCol("cells") .setMargin(3) .setStrip(True) .setOutputCol("table") # Erase detected table regions val fill_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("region") .setOutputCol("image_1") .setRectColor(Color.white) .setFilledRect(True) # OCR val ocr = new ImageToText() .setInputCol("image_1") .setOutputCol("text") .setOcrParams(Array("preserve_interword_spaces=1", )) .setKeepLayout(True) .setOutputSpaceCharacterWidth(8) val pipeline_table = new PipelineModel().setStages(Array( binary_to_image, table_detector, draw_regions, fill_regions, splitter, cell_detector, table_recognition, ocr)) val imagePath = "/content/cTDaR_t10096.jpg" val df = spark.read.format("binaryFile").load(imagePath) val tables_results = pipeline_table.transform(df).cache() ```

## Example ### Input Image ![Screenshot](/assets/images/examples_ocr/table1_1_doc.png) ### Table Structure Recognition {%- capture td_image -%} ![Screenshot](/assets/images/examples_ocr/table1_2_detection.png) {%- endcapture -%} {%- capture tsr_detection -%} ![Screenshot](/assets/images/examples_ocr/table1_3_structure.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=td_image output_image=tsr_detection %} ## Model Information {:.table-model} |---|---| |Model Name:|table_recognition| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_base_finetuned_scrambled_squad_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-10` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_10_en_4.3.0_3.0_1674216712953.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_10_en_4.3.0_3.0_1674216712953.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_scrambled_squad_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-10 --- layout: model title: Legal Natural And Applied Sciences Document Classifier (EURLEX) author: John Snow Labs name: legclf_natural_and_applied_sciences_bert date: 2023-03-06 tags: [en, legal, classification, clauses, natural_and_applied_sciences, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_natural_and_applied_sciences_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Natural_and_Applied_Sciences or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Natural_and_Applied_Sciences`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_natural_and_applied_sciences_bert_en_1.0.0_3.0_1678111577098.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_natural_and_applied_sciences_bert_en_1.0.0_3.0_1678111577098.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_natural_and_applied_sciences_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Natural_and_Applied_Sciences]| |[Other]| |[Other]| |[Natural_and_Applied_Sciences]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_natural_and_applied_sciences_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Natural_and_Applied_Sciences 0.95 0.90 0.92 99 Other 0.90 0.95 0.92 91 accuracy - - 0.92 190 macro-avg 0.92 0.92 0.92 190 weighted-avg 0.92 0.92 0.92 190 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Li) author: John Snow Labs name: roberta_qa_li_base_squad2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `Li`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_li_base_squad2_en_4.3.0_3.0_1674218901284.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_li_base_squad2_en_4.3.0_3.0_1674218901284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_li_base_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_li_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_li_base_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|462.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Li/roberta-base-squad2 - https://rajpurkar.github.io/SQuAD-explorer - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Explain Document pipeline for Danish (explain_document_lg) author: John Snow Labs name: explain_document_lg date: 2021-03-23 tags: [open_source, danish, explain_document_lg, pipeline, da] supported: true task: [Named Entity Recognition, Lemmatization] language: da edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_lg_da_3.0.0_3.0_1616524893608.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_lg_da_3.0.0_3.0_1616524893608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_lg', lang = 'da') annotations = pipeline.fullAnnotate(""Hej fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_lg", lang = "da") val result = pipeline.fullAnnotate("Hej fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hej fra John Snow Labs! ""] result_df = nlu.load('da.explain.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hej fra John Snow Labs! '] | ['Hej fra John Snow Labs!'] | ['Hej', 'fra', 'John', 'Snow', 'Labs!'] | ['Hej', 'fra', 'John', 'Snow', 'Labs!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[-0.025171000510454,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|da| --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6_en_4.0.0_3.0_1655731850822.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6_en_4.0.0_3.0_1655731850822.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|415.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-6 --- layout: model title: Translate English to Vietnamese Pipeline author: John Snow Labs name: translate_en_vi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, vi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `vi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_vi_xx_2.7.0_2.4_1609698576382.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_vi_xx_2.7.0_2.4_1609698576382.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_vi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_vi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.vi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_vi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Arabic ElectraForQuestionAnswering model (from aymanm419) Version-2 author: John Snow Labs name: electra_qa_araElectra_SQUAD_ARCD_768 date: 2022-06-22 tags: [ar, open_source, electra, question_answering] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `araElectra-SQUAD-ARCD-768` is a Arabic model originally trained by `aymanm419`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_araElectra_SQUAD_ARCD_768_ar_4.0.0_3.0_1655920218769.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_araElectra_SQUAD_ARCD_768_ar_4.0.0_3.0_1655920218769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_araElectra_SQUAD_ARCD_768","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_araElectra_SQUAD_ARCD_768","ar") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.squad_arcd.electra.768d").predict("""ما هو اسمي؟|||"اسمي كلارا وأنا أعيش في بيركلي.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_araElectra_SQUAD_ARCD_768| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ar| |Size:|504.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aymanm419/araElectra-SQUAD-ARCD-768 --- layout: model title: Forward-Looking Statements Classification author: John Snow Labs name: finclf_bert_fls date: 2022-09-06 tags: [en, finance, forward, looking, statements, fls, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Text Classification model aimed to detect at sentence or paragraph level, if there is a Forward-looking statements (FLS). FLS are beliefs and opinions about firm's future events or results, usually present in documents as Financial Reports. Identifying forward-looking statements from corporate reports can assist investors in financial analysis. This model was trained originally on 3,500 manually annotated sentences from Management Discussion and Analysis section of annual reports of Russell 3000 firms and then finetuned in house by JSL on low-performant examples. ## Predicted Entities `Specific FLS`, `Non-specific FLS`, `Not FLS` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_bert_fls_en_1.0.0_3.2_1662468990598.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_bert_fls_en_1.0.0_3.2_1662468990598.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+--------------+ | text| result| +--------------------+--------------+ |Global economy wi...|[Specific FLS]| +--------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_bert_fls| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|412.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotations on 10K financial reports and reports from Russell 3000 firms ## Benchmarking ```bash label precision recall f1-score support Specific_FLS 0.96 0.93 0.94 311 Non-specific_FLS 0.91 0.94 0.92 215 Not_FLS 0.84 0.87 0.85 70 accuracy - - 0.92 596 macro-avg 0.90 0.91 0.91 596 weighted-avg 0.93 0.92 0.92 596 ``` --- layout: model title: Pipeline to Detect PHI for Deidentification in Romanian (Word2Vec) author: John Snow Labs name: ner_deid_subentity_pipeline date: 2023-03-09 tags: [ner, deidentification, word2vec, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_subentity](https://nlp.johnsnowlabs.com/2022/06/27/ner_deid_w2v_subentity_ro_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pipeline_ro_4.3.0_3.2_1678386065654.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_pipeline_ro_4.3.0_3.2_1678386065654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_subentity_pipeline", "ro", "clinical/models") text = '''Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui, 737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_subentity_pipeline", "ro", "clinical/models") val text = "Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui, 737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401" val result = pipeline.fullAnnotate(text) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilroberta_base_squad_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_distilroberta_base_squad_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.distilled_base_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilroberta_base_squad_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/squirro/distilroberta-base-squad_v2 - https://paperswithcode.com/sota?task=Question+Answering&dataset=The+Stanford+Question+Answering+Dataset - https://www.linkedin.com/showcase/the-squirro-academy - https://twitter.com/Squirro - https://www.instagram.com/squirro/ - http://squirro.com - https://www.linkedin.com/company/squirroag - https://www.facebook.com/squirro - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Portuguese Large Legal Bert Embeddings author: John Snow Labs name: bert_embeddings_bert_large_cased_pt_lenerbr date: 2022-04-11 tags: [bert, embeddings, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-cased-pt-lenerbr` is a Portuguese model orginally trained by `pierreguillou`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_cased_pt_lenerbr_pt_3.4.2_3.0_1649673910638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_cased_pt_lenerbr_pt_3.4.2_3.0_1649673910638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_cased_pt_lenerbr| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/pierreguillou/bert-large-cased-pt-lenerbr - https://medium.com/@pierre_guillou/nlp-modelos-e-web-app-para-reconhecimento-de-entidade-nomeada-ner-no-dom%C3%ADnio-jur%C3%ADdico-b658db55edfb - https://github.com/piegu/language-models/blob/master/Finetuning_language_model_BERtimbau_LeNER_Br.ipynb - https://paperswithcode.com/sota?task=Fill+Mask&dataset=pierreguillou%2Flener_br_finetuning_language_model --- layout: model title: Pipeline to Detect Cellular/Molecular Biology Entities author: John Snow Labs name: bert_token_classifier_ner_cellular_pipeline date: 2022-03-09 tags: [cellular, ner, bert_for_token_classifier, en, licensed, clinical] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_cellular](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_cellular_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CELLULAR/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CELLULAR.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_pipeline_en_3.4.1_3.0_1646826493144.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_pipeline_en_3.4.1_3.0_1646826493144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline cellular_pipeline = PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") cellular_pipeline.fullAnnotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val cellular_pipeline = new PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") cellular_pipeline.fullAnnotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.cellular_pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash +-----------------------------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------------------------+---------+ |intracellular signaling proteins |protein | |human T-cell leukemia virus type 1 promoter |DNA | |Tax |protein | |Tax-responsive element 1 |DNA | |cyclic AMP-responsive members |protein | |CREB/ATF family |protein | |transcription factors |protein | |Tax |protein | |human T-cell leukemia virus type 1 Tax-responsive element 1|DNA | |TRE-1 |DNA | |lacZ gene |DNA | |CYC1 promoter |DNA | |TRE-1 |DNA | |cyclic AMP response element-binding protein |protein | |CREB |protein | |CREB |protein | |GAL4 activation domain |protein | |GAD |protein | |reporter gene |DNA | |Tax |protein | +-----------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.4 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForTokenClassifier - NerConverter - Finisher --- layout: model title: Classifier for Genders - BIOBERT author: John Snow Labs name: classifierdl_gender_biobert date: 2020-12-16 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.6.5 spark_version: 2.4 tags: [classifier, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model classifies the gender of the patient in the clinical document. {:.h2_title} ## Predicted Entities `Female`, ``Male`, `Unknown`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CLINICAL_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_biobert_en_2.6.4_2.4_1608119684447.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_biobert_en_2.6.4_2.4_1608119684447.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use To classify your text, you can use this model as part of an nlp pipeline with the following stages: DocumentAssembler, BertSentenceEmbeddings (`biobert_pubmed_base_cased`), ClassifierDLModel.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... biobert_embeddings = BertEmbeddings().pretrained('biobert_pubmed_base_cased') \ .setInputCols(["document","token"])\ .setOutputCol("bert_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["document", "bert_embeddings"]) \ .setOutputCol("sentence_bert_embeddings") \ .setPoolingStrategy("AVERAGE") genderClassifier = ClassifierDLModel.pretrained('classifierdl_gender_biobert', 'en', 'clinical/models') \ .setInputCols(["document", "sentence_bert_embeddings"]) \ .setOutputCol("gender") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, biobert_embeddings, sentence_embeddings, gender_classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("""social history: shows that does not smoke cigarettes or drink alcohol, lives in a nursing home. family history: shows a family history of breast cancer.""") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val biobert_embeddings = BertEmbeddings().pretrained("biobert_pubmed_base_cased") .setInputCols(Array("document","token")) .setOutputCol("bert_embeddings") val sentence_embeddings = SentenceEmbeddings() .setInputCols(Array("document", "bert_embeddings")) .setOutputCol("sentence_bert_embeddings") .setPoolingStrategy("AVERAGE") val genderClassifier = ClassifierDLModel.pretrained("classifierdl_gender_biobert", "en", "clinical/models") .setInputCols(Array("document", "sentence_bert_embeddings")) .setOutputCol("gender") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, biobert_embeddings, sentence_embeddings, gender_classifier)) val data = Seq("""social history: shows that does not smoke cigarettes or drink alcohol, lives in a nursing home. family history: shows a family history of breast cancer.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.gender.biobert").predict("""social history: shows that does not smoke cigarettes or drink alcohol, lives in a nursing home. family history: shows a family history of breast cancer.""") ```

{:.h2_title} ## Results ```bash Female ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_gender_biobert| |Type:|ClassifierDLModel| |Compatibility:|Healthcare NLP 2.6.5 +| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|[en]| |Case sensitive:|True| {:.h2_title} ## Data Source This model is trained on more than four thousands clinical documents (radiology reports, pathology reports, clinical visits etc.), annotated internally. {:.h2_title} ## Benchmarking ```bash label precision recall f1-score support Female 0.9224 0.8954 0.9087 239 Male 0.7895 0.8468 0.8171 124 Unknown 0.8077 0.7778 0.7925 54 accuracy 0.8657 417 macro-avg 0.8399 0.8400 0.8394 417 weighted-avg 0.8680 0.8657 0.8664 417 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Celtic Languages author: John Snow Labs name: opus_mt_en_cel date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, cel, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `cel` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_cel_xx_2.7.0_2.4_1609163584650.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_cel_xx_2.7.0_2.4_1609163584650.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_cel| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from mrm8488) author: John Snow Labs name: roberta_qa_base_1b_1_finetuned_squadv2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-1B-1-finetuned-squadv2` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_1b_1_finetuned_squadv2_en_4.2.4_3.0_1669985564517.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_1b_1_finetuned_squadv2_en_4.2.4_3.0_1669985564517.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_1b_1_finetuned_squadv2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_1b_1_finetuned_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_1b_1_finetuned_squadv2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|447.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/roberta-base-1B-1-finetuned-squadv2 - https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: Translate Polish to English Pipeline author: John Snow Labs name: translate_pl_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, pl, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `pl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_pl_en_xx_2.7.0_2.4_1609690625108.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_pl_en_xx_2.7.0_2.4_1609690625108.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_pl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_pl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.pl.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_pl_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal NER for NDA (Remedies Clauses) author: John Snow Labs name: legner_nda_remedies date: 2023-04-16 tags: [en, licensed, ner, legal, nda, remedies] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `REMEDIES` clause with a proper classifier (use `legmulticlf_mnda_sections_paragraph_other` for that purpose). It will extract the following entities: `CURRENCY`, `NUMERIC_REMEDY`, and `REMEDY_TYPE`. ## Predicted Entities `CURRENCY`, `NUMERIC_REMEDY`, `REMEDY_TYPE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_remedies_en_1.0.0_3.0_1681687124993.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_remedies_en_1.0.0_3.0_1681687124993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_remedies", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""The breaching party shall pay the non-breaching party liquidated damages of $ 1,000 per day for each day that the breach of this NDA continues."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +------------------+--------------+ |chunk |ner_label | +------------------+--------------+ |liquidated damages|REMEDY_TYPE | |$ |CURRENCY | |1,000 |NUMERIC_REMEDY| +------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_remedies| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support CURRENCY 1.00 1.00 1.00 11 NUMERIC_REMEDY 1.00 1.00 1.00 11 REMEDY_TYPE 0.86 0.94 0.90 32 micro-avg 0.91 0.96 0.94 54 macro-avg 0.95 0.98 0.97 54 weighted-avg 0.92 0.96 0.94 54 ``` --- layout: model title: Fast Neural Machine Translation Model from Atlantic-Congo Languages to English author: John Snow Labs name: opus_mt_alv_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, alv, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `alv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_alv_en_xx_2.7.0_2.4_1609169974046.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_alv_en_xx_2.7.0_2.4_1609169974046.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_alv_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Social Determinants of Health Mentions author: John Snow Labs name: ner_sdoh_mentions_pipeline date: 2023-03-08 tags: [en, licensed, ner, sdoh, mentions, clinical] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_sdoh_mentions](https://nlp.johnsnowlabs.com/2022/12/18/ner_sdoh_mentions_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_mentions_pipeline_en_4.3.0_3.2_1678281267173.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_mentions_pipeline_en_4.3.0_3.2_1678281267173.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_sdoh_mentions_pipeline", "en", "clinical/models") text = '''Mr. Known lastname 9880 is a pleasant, cooperative gentleman with a long standing history (20 years) diverticulitis. He is married and has 3 children. He works in a bank. He denies any alcohol or intravenous drug use. He has been smoking for many years.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_sdoh_mentions_pipeline", "en", "clinical/models") val text = "Mr. Known lastname 9880 is a pleasant, cooperative gentleman with a long standing history (20 years) diverticulitis. He is married and has 3 children. He works in a bank. He denies any alcohol or intravenous drug use. He has been smoking for many years." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:-----------------|--------:|------:|:-----------------|-------------:| | 0 | married | 123 | 129 | sdoh_community | 0.9972 | | 1 | children | 141 | 148 | sdoh_community | 0.9999 | | 2 | works | 154 | 158 | sdoh_economics | 0.9995 | | 3 | alcohol | 185 | 191 | behavior_alcohol | 0.9925 | | 4 | intravenous drug | 196 | 211 | behavior_drug | 0.9803 | | 5 | smoking | 230 | 236 | behavior_tobacco | 0.9997 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_mentions_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Yoruba Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_yoruba date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, yo, open_source] task: Named Entity Recognition language: yo edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-swahili-finetuned-ner-yoruba` is a Yoruba model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_yoruba_yo_3.4.2_3.0_1652808736465.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_yoruba_yo_3.4.2_3.0_1652808736465.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_yoruba| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|yo| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-swahili-finetuned-ner-yoruba - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukma --- layout: model title: NER on Capital Calls (Small) author: John Snow Labs name: finner_capital_calls date: 2023-02-01 tags: [capital, calls, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a `small` capital call NER, trained to extract contact and financial information from Capital Call Notices. These are the entities retrieved by the model: ``` Financial information: FUND: Name of the Fund called ORG: Organization asking the Fund for the Capital AMOUNT: Amount called by ORG to FUND DUE_DATE: Due date of the call ACCOUNT_NAME: Organization's Bank Account Name ACCOUNT_NUMBER: Organization's Bank Account Number ABA: Routing Number (ABA) BANK_ADDRESS: Contact address of the Bank Contact information: PHONE: Contact Phone PERSON: Contact Person BANK_CONTACT: Person to contact in Bank EMAIL: Contact Email Other additional information, not directly involved in the call: OTHER_PERSON: Other people detected (People signing the call, people to whom is addressed the Notice, etc) OTHER_PERCENTAGE: Percentages mentiones OTHER_DATE: Other dates mentioned, not Due Date OTHER_AMOUNT: Other amounts mentioned OTHER_ADDRESS: Other addresses mentiones OTHER_ORG: Other ORG mentiones ``` ## Predicted Entities `FUND`, `ORG`, `AMOUNT`, `DUE_DATE`, `ACCOUNT_NAME`, `ACCOUNT_NUMBER`, `BANK_ADDRESS`, `PHONE`, `PERSON`, `BANK_CONTACT`, `EMAIL`, `OTHER_PERSON`, `OTHER_PERCENTAGE`, `OTHER_DATE`, `OTHER_AMOUNT`, `OTHER_ADDRESS`, `OTHER_ORG`, `ABA` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_CAPITAL_CALLS){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_capital_calls_en_1.0.0_3.0_1675250939298.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_capital_calls_en_1.0.0_3.0_1675250939298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from pyspark.sql import functions as F documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = nlp.Tokenizer() \ .setInputCols("sentence") \ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner = finance.NerModel.pretrained('finner_capital_calls', 'en', 'finance/models')\ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") converter = finance.NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ pipeline = nlp.Pipeline(stages=[documentAssembler, sentence, tokenizer, embeddings, ner, converter ]) df = spark.createDataFrame([[""]]).toDF("text") model = pipeline.fit(df) lp = nlp.LightPipeline(model) text = """Dear Charlotte R. Davis, We hope this message finds you well. This is to inform you that a capital call for Upfront Ventures has been initiated. The amount requested is 7000 EUR and is due on 01.01.2024. Kindly wire transfer the funds to the following account: Account Green Planet Solutions LLC Account Number 1234567-1XX Routing Number 51903761 Bank First Republic Bank If you require any further information, please do not hesitate to reach out to us at 3055 550818 or coxeric@example.com. Thank you for your prompt attention to this matter. Best regards, James Wilson""" result = model.transform(spark.createDataFrame([[text]]).toDF("text")) from pyspark.sql import functions as F result.select(F.explode(F.arrays_zip(result.ner_chunk.result, result.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['0']").alias("chunk"), F.expr("cols['1']['entity']").alias("ner_label"), F.expr("cols['1']['confidence']").alias("confidence")).show(truncate=False) ```

## Results ```bash +--------------------------+--------------+----------+ |chunk |ner_label |confidence| +--------------------------+--------------+----------+ |Charlotte R. Davis |OTHER_PERSON |0.971875 | |Upfront Ventures |FUND |1.0 | |7000 EUR |AMOUNT |1.0 | |01.01.2024 |DUE_DATE |1.0 | |Green Planet Solutions LLC|ACCOUNT_NAME |0.999875 | |1234567-1XX |ACCOUNT_NUMBER|1.0 | |51903761 |ABA |1.0 | |First Republic Bank |BANK_NAME |0.9999333 | |3055 550818 |PHONE |1.0 | |coxeric@example.com |EMAIL |1.0 | |James Wilson |OTHER_PERSON |1.0 | +--------------------------+--------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_capital_calls| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house capital call notices ## Benchmarking ```bash Total test loss: 0.3542 Avg test loss: 0.0295 label tp fp fn prec rec f1 B-PERSON 47 0 0 1.0 1.0 1.0 I-OTHER_PERSON 214 0 0 1.0 1.0 1.0 I-AMOUNT 127 0 0 1.0 1.0 1.0 I-OTHER_PERCENTAGE 37 0 0 1.0 1.0 1.0 B-OTHER_DATE 25 0 0 1.0 1.0 1.0 I-BANK_ADDRESS 121 0 0 1.0 1.0 1.0 B-AMOUNT 170 0 0 1.0 1.0 1.0 B-OTHER_AMOUNT 409 0 0 1.0 1.0 1.0 I-ORG 311 18 0 0.9452888 1.0 0.971875 B-PHONE 79 0 0 1.0 1.0 1.0 I-DUE_DATE 153 0 0 1.0 1.0 1.0 B-FUND 124 0 0 1.0 1.0 1.0 B-ABA 97 0 0 1.0 1.0 1.0 I-ACCOUNT_NAME 223 0 0 1.0 1.0 1.0 I-OTHER_DATE 25 0 0 1.0 1.0 1.0 I-PHONE 119 0 0 1.0 1.0 1.0 B-BANK_ADDRESS 39 0 0 1.0 1.0 1.0 B-OTHER_ORG 139 0 6 1.0 0.95862067 0.97887325 I-OTHER_AMOUNT 307 0 0 1.0 1.0 1.0 I-FUND 131 0 0 1.0 1.0 1.0 I-BANK_NAME 139 0 0 1.0 1.0 1.0 B-EMAIL 73 0 0 1.0 1.0 1.0 I-BANK_CONTACT 52 0 0 1.0 1.0 1.0 B-BANK_CONTACT 30 0 0 1.0 1.0 1.0 B-OTHER_PERSON 116 0 0 1.0 1.0 1.0 B-ACCOUNT_NAME 97 0 0 1.0 1.0 1.0 B-DUE_DATE 127 0 0 1.0 1.0 1.0 B-OTHER_ADDRESS 11 0 0 1.0 1.0 1.0 B-ORG 147 6 0 0.9607843 1.0 0.98 B-BANK_NAME 113 0 0 1.0 1.0 1.0 B-OTHER_PERCENTAGE 74 0 0 1.0 1.0 1.0 I-OTHER_ADDRESS 38 0 0 1.0 1.0 1.0 B-ACCOUNT_NUMBER 97 0 0 1.0 1.0 1.0 I-PERSON 109 0 0 1.0 1.0 1.0 I-OTHER_ORG 283 0 18 1.0 0.9401993 0.969178 I-ACCOUNT_NUMBER 32 0 0 1.0 1.0 1.0 Macro-average 4435 24 24 0.997391 0.9971894 0.9972902 Micro-average 4435 24 24 0.99461764 0.99461764 0.99461764 ``` --- layout: model title: English LongformerForQuestionAnswering model (from manishiitg) author: John Snow Labs name: longformer_qa_recruit date: 2022-06-26 tags: [en, open_source, longformer, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `longformer-recruit-qa` is a English model originally trained by `manishiitg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_qa_recruit_en_4.0.0_3.0_1656255562609.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_qa_recruit_en_4.0.0_3.0_1656255562609.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_qa_recruit","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_qa_recruit","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.longformer.by_manishiitg").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_qa_recruit| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|556.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/manishiitg/longformer-recruit-qa --- layout: model title: English BertForTokenClassification Cased model (from nguyenkhoa2407) author: John Snow Labs name: bert_token_classifier_autotrain_ner_favsbot date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-bert-NER-favsbot` is a English model originally trained by `nguyenkhoa2407`. ## Predicted Entities `TIME`, `SORT`, `PER`, `LOC`, `TAG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_ner_favsbot_en_4.2.4_3.0_1669814444554.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_ner_favsbot_en_4.2.4_3.0_1669814444554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_ner_favsbot","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_ner_favsbot","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autotrain_ner_favsbot| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nguyenkhoa2407/autotrain-bert-NER-favsbot --- layout: model title: Adverse Drug Events Classifier (DistilBERT) author: John Snow Labs name: distilbert_sequence_classifier_ade date: 2022-02-08 tags: [bert, sequence_classification, en, licensed] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: MedicalDistilBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify text/sentence in two categories: `True` : The sentence is talking about a possible ADE `False` : The sentences doesn’t have any information about an ADE. This model is a [DistilBERT](https://huggingface.co/distilbert-base-cased)-based classifier. Please note that there is no bio-version of DistilBERT so the performance may not be par with BioBERT-based classifiers. ## Predicted Entities `True`, `False` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_ADE/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/08.3.MedicalBertForSequenceClassification_in_SparkNLP.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/distilbert_sequence_classifier_ade_en_3.4.1_3.0_1644352732829.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/distilbert_sequence_classifier_ade_en_3.4.1_3.0_1644352732829.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------+------+ |text |result| +----------------------------------------------------------------+------+ |I felt a bit drowsy and had blurred vision after taking Aspirin.|[True]| +----------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_sequence_classifier_ade| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model is trained on a custom dataset comprising of CADEC, DRUG-AE and Twimed. ## Benchmarking ```bash label precision recall f1-score support False 0.93 0.93 0.93 6935 True 0.64 0.65 0.65 1347 accuracy 0.88 0.88 0.88 8282 macro-avg 0.79 0.79 0.79 8282 weighted-avg 0.89 0.88 0.89 8282 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from anablasi) author: John Snow Labs name: roberta_qa_model_10k date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `model_10k_qa` is a English model originally trained by `anablasi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_model_10k_en_4.3.0_3.0_1674211482662.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_model_10k_en_4.3.0_3.0_1674211482662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_model_10k","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_model_10k","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_model_10k| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anablasi/model_10k_qa --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-128-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6_en_4.0.0_3.0_1657184326520.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6_en_4.0.0_3.0_1657184326520.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_128_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-128-finetuned-squad-seed-6 --- layout: model title: English DistilBertForQuestionAnswering model (from threem) Squad1 author: John Snow Labs name: distilbert_qa_mysquadv2_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mysquadv2-finetuned-squad` is a English model originally trained by `threem`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_mysquadv2_finetuned_squad_en_4.0.0_3.0_1654728438175.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_mysquadv2_finetuned_squad_en_4.0.0_3.0_1654728438175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mysquadv2_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mysquadv2_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.distil_bert.by_threem").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_mysquadv2_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/threem/mysquadv2-finetuned-squad --- layout: model title: Detect Oncology-Specific Entities author: John Snow Labs name: ner_oncology_limited_80p_for_benchmarks date: 2023-04-03 tags: [licensed, clinical, en, oncology, biomarker, treatment] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description `Important Note:` This model is trained with a partial dataset that is used to train [ner_oncology](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_en.html); and meant to be used for benchmarking run at [LLMs Healthcare Benchmarks](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/academic/LLMs_in_Healthcare). This model extracts more than 40 oncology-related entities, including therapies, tests and staging. Definitions of Predicted Entities: `Adenopathy`: Mentions of pathological findings of the lymph nodes. `Age`: All mention of ages, past or present, related to the patient or with anybody else. `Biomarker`: Biological molecules that indicate the presence or absence of cancer, or the type of cancer. Oncogenes are excluded from this category. `Biomarker_Result`: Terms or values that are identified as the result of a biomarkers. `Cancer_Dx`: Mentions of cancer diagnoses (such as “breast cancer”) or pathological types that are usually used as synonyms for “cancer” (e.g. “carcinoma”). When anatomical references are present, they are included in the Cancer_Dx extraction. `Cancer_Score`: Clinical or imaging scores that are specific for cancer settings (e.g. “BI-RADS” or “Allred score”). `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. `Chemotherapy`: Mentions of chemotherapy drugs, or unspecific words such as “chemotherapy”. `Cycle_Coun`: The total number of cycles being administered of an oncological therapy (e.g. “5 cycles”). `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. “day 5”). `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. “third cycle”). `Date`: Mentions of exact dates, in any format, including day number, month and/or year. `Death_Entity`: Words that indicate the death of the patient or someone else (including family members), such as “died” or “passed away”. `Direction`: Directional and laterality terms, such as “left”, “right”, “bilateral”, “upper” and “lower”. `Dosage`: The quantity prescribed by the physician for an active ingredient. `Duration`: Words indicating the duration of a treatment (e.g. “for 2 weeks”). `Frequency`: Words indicating the frequency of treatment administration (e.g. “daily” or “bid”). `Gender`: Gender-specific nouns and pronouns (including words such as “him” or “she”, and family members such as “father”). `Grade`: All pathological grading of tumors (e.g. “grade 1”) or degrees of cellular differentiation (e.g. “well-differentiated”) `Histological_Type`: Histological variants or cancer subtypes, such as “papillary”, “clear cell” or “medullary”. `Hormonal_Therapy`: Mentions of hormonal drugs used to treat cancer, or unspecific words such as “hormonal therapy”. `Imaging_Test`: Imaging tests mentioned in texts, such as “chest CT scan”. `Immunotherapy`: Mentions of immunotherapy drugs, or unspecific words such as “immunotherapy”. `Invasion`: Mentions that refer to tumor invasion, such as “invasion” or “involvement”. Metastases or lymph node involvement are excluded from this category. `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. “first-line treatment”). `Metastasis`: Terms that indicate a metastatic disease. Anatomical references are not included in these extractions. `Oncogene`: Mentions of genes that are implicated in the etiology of cancer. `Pathology_Result`: The findings of a biopsy from the pathology report that is not covered by another entity (e.g. “malignant ductal cells”). `Pathology_Test`: Mentions of biopsies or tests that use tissue samples. `Performance_Status`: Mentions of performance status scores, such as ECOG and Karnofsky. The name of the score is extracted together with the result (e.g. “ECOG performance status of 4”). `Race_Ethnicity`: The race and ethnicity categories include racial and national origin or sociocultural groups. `Radiotherapy`: Terms that indicate the use of Radiotherapy. `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including “recurrence”, “bad response” or “improvement”. `Relative_Date`: Temporal references that are relative to the date of the text or to any other specific date (e.g. “yesterday” or “three years later”). `Route`: Words indicating the type of administration route (such as “PO” or “transdermal”). `Site_Bone`: Anatomical terms that refer to the human skeleton. `Site_Brain`: Anatomical terms that refer to the central nervous system (including the brain stem and the cerebellum). `Site_Breast`: Anatomical terms that refer to the breasts. `Site_Liver`: Anatomical terms that refer to the liver. `Site_Lung`: Anatomical terms that refer to the lungs. `Site_Lymph_Node`: Anatomical terms that refer to lymph nodes, excluding adenopathies. `Site_Other_Body_Part`: Relevant anatomical terms that are not included in the rest of the anatomical entities. `Smoking_Status`: All mentions of smoking related to the patient or to someone else. `Staging`: Mentions of cancer stage such as “stage 2b” or “T2N1M0”. It also includes words such as “in situ”, “early-stage” or “advanced”. `Targeted_Therapy`: Mentions of targeted therapy drugs, or unspecific words such as “targeted therapy”. `Tumor_Finding`: All nonspecific terms that may be related to tumors, either malignant or benign (for example: “mass”, “tumor”, “lesion”, or “neoplasm”). `Tumor_Size`: Size of the tumor, including numerical value and unit of measurement (e.g. “3 cm”). `Unspecific_Therapy`: Terms that indicate a known cancer therapy but that is not specific to any other therapy entity (e.g. “chemoradiotherapy” or “adjuvant therapy”). ## Predicted Entities `Histological_Type`, `Direction`, `Staging`, `Cancer_Score`, `Imaging_Test`, `Cycle_Number`, `Tumor_Finding`, `Site_Lymph_Node`, `Invasion`, `Response_To_Treatment`, `Smoking_Status`, `Tumor_Size`, `Cycle_Count`, `Adenopathy`, `Age`, `Biomarker_Result`, `Unspecific_Therapy`, `Site_Breast`, `Chemotherapy`, `Targeted_Therapy`, `Radiotherapy`, `Performance_Status`, `Pathology_Test`, `Site_Other_Body_Part`, `Cancer_Surgery`, `Line_Of_Therapy`, `Pathology_Result`, `Hormonal_Therapy`, `Site_Bone`, `Biomarker`, `Immunotherapy`, `Cycle_Day`, `Frequency`, `Route`, `Duration`, `Death_Entity`, `Metastasis`, `Site_Liver`, `Cancer_Dx`, `Grade`, `Date`, `Site_Lung`, `Site_Brain`, `Relative_Date`, `Race_Ethnicity`, `Gender`, `Oncogene`, `Dosage`, `Radiation_Dose` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_limited_80p_for_benchmarks_en_4.3.2_3.0_1680548141397.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_limited_80p_for_benchmarks_en_4.3.2_3.0_1680548141397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+-----+---+---------------------+ |chunk |begin|end|ner_label | +------------------------------+-----+---+---------------------+ |left |31 |34 |Direction | |mastectomy |36 |45 |Cancer_Surgery | |axillary lymph node dissection|54 |83 |Cancer_Surgery | |left |91 |94 |Direction | |breast cancer |96 |108|Cancer_Dx | |twenty years ago |110 |125|Relative_Date | |tumor |132 |136|Tumor_Finding | |positive |142 |149|Biomarker_Result | |ER |155 |156|Biomarker | |PR |162 |163|Biomarker | |radiotherapy |183 |194|Radiotherapy | |breast |229 |234|Site_Breast | |cancer |241 |246|Cancer_Dx | |recurred |248 |255|Response_To_Treatment| |right |262 |266|Direction | |lung |268 |271|Site_Lung | |metastasis |273 |282|Metastasis | |13 years later |284 |297|Relative_Date | |adriamycin |346 |355|Chemotherapy | |60 mg/m2 |358 |365|Dosage | |cyclophosphamide |372 |387|Chemotherapy | |600 mg/m2 |390 |398|Dosage | |six courses |406 |416|Cycle_Count | |first line |422 |431|Line_Of_Therapy | +------------------------------+-----+---+---------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_limited_80p_for_benchmarks| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.3 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Histological_Type 135.0 60.0 68.0 203.0 0.6923 0.665 0.6784 Direction 642.0 170.0 149.0 791.0 0.7906 0.8116 0.801 Staging 122.0 20.0 29.0 151.0 0.8592 0.8079 0.8328 Cancer_Score 14.0 8.0 8.0 22.0 0.6364 0.6364 0.6364 Imaging_Test 743.0 197.0 155.0 898.0 0.7904 0.8274 0.8085 Cycle_Number 45.0 34.0 10.0 55.0 0.5696 0.8182 0.6716 Tumor_Finding 948.0 104.0 110.0 1058.0 0.9011 0.896 0.8986 Site_Lymph_Node 199.0 62.0 57.0 256.0 0.7625 0.7773 0.7698 Invasion 116.0 33.0 20.0 136.0 0.7785 0.8529 0.814 Response_To_Treat... 244.0 145.0 144.0 388.0 0.6272 0.6289 0.6281 Smoking_Status 54.0 7.0 2.0 56.0 0.8852 0.9643 0.9231 Cycle_Count 96.0 26.0 32.0 128.0 0.7869 0.75 0.768 Tumor_Size 205.0 43.0 49.0 254.0 0.8266 0.8071 0.8167 Adenopathy 29.0 6.0 4.0 33.0 0.8286 0.8788 0.8529 Age 212.0 14.0 18.0 230.0 0.9381 0.9217 0.9298 Biomarker_Result 593.0 138.0 122.0 715.0 0.8112 0.8294 0.8202 Unspecific_Therapy 124.0 47.0 50.0 174.0 0.7251 0.7126 0.7188 Site_Breast 96.0 13.0 14.0 110.0 0.8807 0.8727 0.8767 Chemotherapy 570.0 40.0 65.0 635.0 0.9344 0.8976 0.9157 Targeted_Therapy 173.0 11.0 17.0 190.0 0.9402 0.9105 0.9251 Radiotherapy 128.0 26.0 21.0 149.0 0.8312 0.8591 0.8449 Performance_Status 29.0 10.0 14.0 43.0 0.7436 0.6744 0.7073 Pathology_Test 395.0 157.0 105.0 500.0 0.7156 0.79 0.751 Site_Other_Body_Part 682.0 284.0 370.0 1052.0 0.706 0.6483 0.6759 Cancer_Surgery 388.0 100.0 75.0 463.0 0.7951 0.838 0.816 Line_Of_Therapy 30.0 9.0 8.0 38.0 0.7692 0.7895 0.7792 Pathology_Result 135.0 162.0 169.0 304.0 0.4545 0.4441 0.4493 Hormonal_Therapy 95.0 9.0 10.0 105.0 0.9135 0.9048 0.9091 Site_Bone 171.0 42.0 68.0 239.0 0.8028 0.7155 0.7566 Immunotherapy 57.0 31.0 13.0 70.0 0.6477 0.8143 0.7215 Biomarker 759.0 161.0 118.0 877.0 0.825 0.8655 0.8447 Cycle_Day 84.0 32.0 32.0 116.0 0.7241 0.7241 0.7241 Frequency 199.0 33.0 32.0 231.0 0.8578 0.8615 0.8596 Route 88.0 12.0 23.0 111.0 0.88 0.7928 0.8341 Duration 184.0 60.0 110.0 294.0 0.7541 0.6259 0.684 Death_Entity 36.0 3.0 2.0 38.0 0.9231 0.9474 0.9351 Metastasis 307.0 18.0 21.0 328.0 0.9446 0.936 0.9403 Site_Liver 138.0 46.0 35.0 173.0 0.75 0.7977 0.7731 Cancer_Dx 720.0 112.0 100.0 820.0 0.8654 0.878 0.8717 Grade 48.0 21.0 13.0 61.0 0.6957 0.7869 0.7385 Date 365.0 17.0 16.0 381.0 0.9555 0.958 0.9567 Site_Lung 354.0 100.0 87.0 441.0 0.7797 0.8027 0.7911 Site_Brain 133.0 28.0 59.0 192.0 0.8261 0.6927 0.7535 Relative_Date 365.0 264.0 80.0 445.0 0.5803 0.8202 0.6797 Race_Ethnicity 51.0 10.0 5.0 56.0 0.8361 0.9107 0.8718 Gender 1267.0 15.0 3.0 1270.0 0.9883 0.9976 0.9929 Dosage 337.0 45.0 78.0 415.0 0.8822 0.812 0.8457 Oncogene 135.0 57.0 78.0 213.0 0.7031 0.6338 0.6667 Radiation_Dose 35.0 10.0 6.0 41.0 0.7778 0.8537 0.814 macro - - - - - - 0.7974 micro - - - - - - 0.8154 ``` --- layout: model title: English asr_wav2vec2_base_960h_4_gram TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: asr_wav2vec2_base_960h_4_gram date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_960h_4_gram` is a English model originally trained by patrickvonplaten. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_960h_4_gram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_4_gram_en_4.2.0_3.0_1664022178956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_4_gram_en_4.2.0_3.0_1664022178956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_960h_4_gram", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_960h_4_gram", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_960h_4_gram| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.6 MB| --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-16-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2_en_4.0.0_3.0_1657184494597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2_en_4.0.0_3.0_1657184494597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-16-finetuned-squad-seed-2 --- layout: model title: Icelandic RobertaForQuestionAnswering Cased model (from vesteinn) author: John Snow Labs name: roberta_qa_icebert date: 2022-12-02 tags: [is, open_source, roberta, question_answering, tensorflow] task: Question Answering language: is edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `IceBERT-QA` is a Icelandic model originally trained by `vesteinn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_is_4.2.4_3.0_1669972802947.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_is_4.2.4_3.0_1669972802947.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_icebert","is") \ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_icebert","is") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_icebert| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|is| |Size:|463.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/vesteinn/IceBERT-QA --- layout: model title: Hindi DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_hi_cased date: 2022-04-12 tags: [distilbert, embeddings, hi, open_source] task: Embeddings language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-hi-cased` is a Hindi model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_hi_cased_hi_3.4.2_3.0_1649783460616.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_hi_cased_hi_3.4.2_3.0_1649783460616.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_hi_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|177.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-hi-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Spanish RobertaForTokenClassification Cased model (from mrm8488) author: John Snow Labs name: roberta_ner_finetuned_bioclinical date: 2022-07-18 tags: [open_source, roberta, ner, bioclinical, es] task: Named Entity Recognition language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa NER model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bioclinical-roberta-es-finenuned-clinical-ner` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_finetuned_bioclinical_es_4.0.0_3.0_1658155068450.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_finetuned_bioclinical_es_4.0.0_3.0_1658155068450.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_finetuned_bioclinical| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|441.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References https://huggingface.co/mrm8488/bioclinical-roberta-es-finenuned-clinical-ner --- layout: model title: Fast Neural Machine Translation Model from Armenian to English author: John Snow Labs name: opus_mt_hy_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, hy, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `hy` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_hy_en_xx_2.7.0_2.4_1609169598231.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_hy_en_xx_2.7.0_2.4_1609169598231.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_hy_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Persian ALBERT Embeddings (from m3hrdadfi) author: John Snow Labs name: albert_embeddings_albert_fa_base_v2 date: 2022-04-14 tags: [albert, embeddings, fa, open_source] task: Embeddings language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-fa-base-v2` is a Persian model orginally trained by `m3hrdadfi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_fa_base_v2_fa_3.4.2_3.0_1649954318874.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_fa_base_v2_fa_3.4.2_3.0_1649954318874.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_fa_base_v2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fa| |Size:|69.2 MB| |Case sensitive:|false| ## References - https://huggingface.co/m3hrdadfi/albert-fa-base-v2 - https://dumps.wikimedia.org/fawiki/ - https://github.com/miras-tech/MirasText - https://bigbangpage.com/ - https://www.chetor.com/ - https://www.eligasht.com/Blog/ - https://www.digikala.com/mag/ - https://www.ted.com/talks - https://github.com/m3hrdadfi/albert-persian - https://github.com/hooshvare/parsbert - https://github.com/m3hrdadfi/albert-persian --- layout: model title: Lemmatizer (Portugese, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-08 tags: [open_source, lemmatizer, pt] task: Lemmatization language: pt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Portugese Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_pt_3.4.1_3.0_1646753629532.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_pt_3.4.1_3.0_1646753629532.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------+ |result | +---------------------------------+ |[Você, não, ser, melhor, que, eu]| +---------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|pt| |Size:|8.9 MB| --- layout: model title: Detect Assertion Status (assertion_dl_en) author: John Snow Labs name: assertion_dl_en date: 2020-01-30 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.4.0 spark_version: 2.4 tags: [clinical, licensed, ner, en] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deep learning named entity recognition model for assertions. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. {:.h2_title} ## Predicted Entities ``hypothetical``, ``present``, ``absent``, ``possible``, ``conditional``, ``associated_with_someone_else``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_en_2.4.0_2.4_1580237286004.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_en_2.4.0_2.4_1580237286004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel, NerConverter, AssertionDLModel.

{% include programmingLanguageSelectScalaPython.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("assertion_dl", "en", "clinical/models") \ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[document_assembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_result = LightPipeline(model).fullAnnotate('Patient has a headache for the last 2 weeks and appears anxious when she walks fast. No alopecia noted. She denies pain')[0] ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("assertion_dl", "en", "clinical/models") .setInputCols(Array("sentence", "ner_chunk", "embeddings")) .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(document_assembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("Patient has a headache for the last 2 weeks and appears anxious when she walks fast. No alopecia noted. She denies pain").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results The output is a dataframe with a sentence per row and an ``"assertion"`` column containing all of the assertion labels in the sentence. The assertion column also contains assertion character indices, and other metadata. To get only the entity chunks and assertion labels, without the metadata, select ``"ner_chunk.result"`` and ``"assertion.result"`` from your output dataframe. ```bash | | chunks | entities | assertion | |---|------------|----------|-------------| | 0 | a headache | PROBLEM | present | | 1 | anxious | PROBLEM | conditional | | 2 | alopecia | PROBLEM | absent | | 3 | pain | PROBLEM | absent | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_dl| |Type:|ner| |Compatibility:|Spark NLP 2.4.0| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence, ner_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|[en]| |Case sensitive:|false| ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash label prec rec f1 absent 0.94 0.87 0.91 associated_with_someone_else 0.81 0.73 0.76 conditional 0.78 0.24 0.37 hypothetical 0.89 0.75 0.81 possible 0.70 0.52 0.60 present 0.91 0.97 0.94 Macro-average 0.84 0.68 0.73 Micro-average 0.91 0.91 0.91 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from ms12345) author: John Snow Labs name: roberta_qa_ms12345_base_squad2_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `ms12345`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ms12345_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219374435.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ms12345_base_squad2_finetuned_squad_en_4.3.0_3.0_1674219374435.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ms12345_base_squad2_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ms12345_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ms12345_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ms12345/roberta-base-squad2-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from American Sign Language to French author: John Snow Labs name: opus_mt_ase_fr date: 2021-06-01 tags: [open_source, seq2seq, translation, ase, fr, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ase target languages: fr {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ase_fr_xx_3.1.0_2.4_1622561105602.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ase_fr_xx_3.1.0_2.4_1622561105602.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ase_fr| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from mrm8488) Xqua author: John Snow Labs name: distilbert_qa_multi_finetuned_for_xqua_on_tydiqa date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-multi-finetuned-for-xqua-on-tydiqa` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_multi_finetuned_for_xqua_on_tydiqa_en_4.0.0_3.0_1654727619668.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_multi_finetuned_for_xqua_on_tydiqa_en_4.0.0_3.0_1654727619668.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_multi_finetuned_for_xqua_on_tydiqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_multi_finetuned_for_xqua_on_tydiqa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.tydiqa.distil_bert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_multi_finetuned_for_xqua_on_tydiqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/distilbert-multi-finetuned-for-xqua-on-tydiqa - https://ai.google.com/research/tydiqa - https://github.com/google-research-datasets/tydiqa/blob/master/README.md#the-tasks - https://twitter.com/mrm8488 --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-32-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10_en_4.0.0_3.0_1657184919799.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10_en_4.0.0_3.0_1657184919799.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-32-finetuned-squad-seed-10 --- layout: model title: English asr_wav2vec2_base_100h_test TFWav2Vec2ForCTC from saahith author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_test date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_test` is a English model originally trained by saahith. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_test_en_4.2.0_3.0_1664094940990.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_test_en_4.2.0_3.0_1664094940990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_test', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_test", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Cased model (from osanseviero) author: John Snow Labs name: t5_finetuned_test date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-finetuned-test` is a English model originally trained by `osanseviero`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_finetuned_test_en_4.3.0_3.0_1675124670488.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_finetuned_test_en_4.3.0_3.0_1675124670488.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_finetuned_test","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_finetuned_test","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_finetuned_test| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|286.9 MB| ## References - https://huggingface.co/osanseviero/t5-finetuned-test - https://medium.com/@priya.dwivedi/fine-tuning-a-t5-transformer-for-any-summarization-task-82334c64c81 --- layout: model title: News Classifier Pipeline for German text author: John Snow Labs name: classifierdl_bert_news_pipeline date: 2021-08-13 tags: [de, classifier, pipeline, news, open_source] task: Pipeline Public language: de edition: Spark NLP 3.1.3 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pre-trained pipeline classifies German texts of news. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_DE_NEWS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_DE_NEWS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_bert_news_pipeline_de_3.1.3_2.4_1628851787696.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_bert_news_pipeline_de_3.1.3_2.4_1628851787696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_bert_news_pipeline", lang = "de") result = pipeline.fullAnnotate("""Niki Lauda in einem McLaren MP 4/2 TAG Turbo. Mit diesem Gefährt sicherte sich der Österreicher 1984 seinen dritten Weltmeistertitel, einen halben (!)""") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_bert_news_pipeline", "de") val result = pipeline.fullAnnotate("Niki Lauda in einem McLaren MP 4/2 TAG Turbo. Mit diesem Gefährt sicherte sich der Österreicher 1984 seinen dritten Weltmeistertitel, einen halben (!)")(0) ```

## Results ```bash ["Sport"] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_news_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.1.3+| |License:|Open Source| |Edition:|Official| |Language:|de| ## Included Models - DocumentAssembler - BertSentenceEmbeddings - ClassifierDLModel --- layout: model title: English DistilBertForTokenClassification Base Cased model (from 51la5) author: John Snow Labs name: distilbert_token_classifier_base_ner date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-NER` is a English model originally trained by `51la5`. ## Predicted Entities `LOC`, `ORG`, `PER`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_ner_en_4.3.1_3.0_1678133783319.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_ner_en_4.3.1_3.0_1678133783319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_ner","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_ner","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_base_ner| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/51la5/distilbert-base-NER - https://paperswithcode.com/sota?task=Token+Classification&dataset=conll2003 --- layout: model title: English ALBERT Embeddings (x-large) author: John Snow Labs name: albert_embeddings_albert_xlarge_v1 date: 2022-04-14 tags: [albert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-xlarge-v1` is a English model orginally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_xlarge_v1_en_3.4.2_3.0_1649954213986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_xlarge_v1_en_3.4.2_3.0_1649954213986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_xlarge_v1| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|221.6 MB| |Case sensitive:|false| ## References - https://huggingface.co/albert-xlarge-v1 - https://arxiv.org/abs/1909.11942 - https://github.com/google-research/albert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: Recognize Entities DL Pipeline for Russian - Medium author: John Snow Labs name: entity_recognizer_md date: 2021-03-22 tags: [open_source, russian, entity_recognizer_md, pipeline, ru] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: ru edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_ru_3.0.0_3.0_1616448672830.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_ru_3.0.0_3.0_1616448672830.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_md', lang = 'ru') annotations = pipeline.fullAnnotate(""Здравствуйте из Джона Снежных Лабораторий! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_md", lang = "ru") val result = pipeline.fullAnnotate("Здравствуйте из Джона Снежных Лабораторий! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Здравствуйте из Джона Снежных Лабораторий! ""] result_df = nlu.load('ru.ner.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:------------------------------------------------|:-----------------------------------------------|:-----------------------------------------------------------|:-----------------------------|:--------------------------------------|:-------------------------------| | 0 | ['Здравствуйте из Джона Снежных Лабораторий! '] | ['Здравствуйте из Джона Снежных Лабораторий!'] | ['Здравствуйте', 'из', 'Джона', 'Снежных', 'Лабораторий!'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-LOC', 'I-LOC', 'I-LOC'] | ['Джона Снежных Лабораторий!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|ru| --- layout: model title: RE Pipeline between Body Parts and Direction Entities author: John Snow Labs name: re_bodypart_directions_pipeline date: 2023-06-13 tags: [licensed, clinical, relation_extraction, body_part, directions, en] task: Relation Extraction language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_bodypart_directions](https://nlp.johnsnowlabs.com/2021/01/18/re_bodypart_directions_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_pipeline_en_4.4.4_3.2_1686664392280.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_pipeline_en_4.4.4_3.2_1686664392280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_bodypart_directions_pipeline", "en", "clinical/models") pipeline.fullAnnotate("MRI demonstrated infarction in the upper brain stem , left cerebellum and right basil ganglia") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_bodypart_directions_pipeline", "en", "clinical/models") pipeline.fullAnnotate("MRI demonstrated infarction in the upper brain stem , left cerebellum and right basil ganglia") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.bodypart_directions.pipeline").predict("""MRI demonstrated infarction in the upper brain stem , left cerebellum and right basil ganglia""") ```

## Results ```bash Results | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|-----------------------------|---------------|-------------|------------|-----------------------------|-------------|-------------|---------------|------------| | 0 | 1 | Direction | 35 | 39 | upper | Internal_organ_or_component | 41 | 50 | brain stem | 0.9999989 | | 1 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 59 | 68 | cerebellum | 0.99992585 | | 2 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.9999999 | | 3 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 54 | 57 | left | 0.999811 | | 4 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 75 | 79 | right | 0.9998203 | | 5 | 1 | Direction | 54 | 57 | left | Internal_organ_or_component | 59 | 68 | cerebellum | 1.0 | | 6 | 0 | Direction | 54 | 57 | left | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.97616416 | | 7 | 0 | Internal_organ_or_component | 59 | 68 | cerebellum | Direction | 75 | 79 | right | 0.953046 | | 8 | 1 | Direction | 75 | 79 | right | Internal_organ_or_component | 81 | 93 | basil ganglia | 1.0 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_bodypart_directions_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: Icelandic DistilBertForTokenClassification Cased model (from m3hrdadfi) author: John Snow Labs name: distilbert_token_classifier_typo_detector date: 2023-03-06 tags: [is, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: is edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `typo-detector-distilbert-is` is a Icelandic model originally trained by `m3hrdadfi`. ## Predicted Entities `TYPO` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dtilbert_token_classifier_typo_detector_is_4.3.1_3.0_1678134296845.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dtilbert_token_classifier_typo_detector_is_4.3.1_3.0_1678134296845.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("dtilbert_token_classifier_typo_detector","is") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("dtilbert_token_classifier_typo_detector","is") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dtilbert_token_classifier_typo_detector| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|is| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m3hrdadfi/typo-detector-distilbert-is - https://github.com/m3hrdadfi/typo-detector/issues --- layout: model title: English asr_wav2vec2_base_timit_demo_colab11_by_sameearif88 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab11_by_sameearif88` is a English model originally trained by sameearif88. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_en_4.2.0_3.0_1664021315565.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88_en_4.2.0_3.0_1664021315565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab11_by_sameearif88| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English XlmRoBertaForQuestionAnswering (from Dongjae) author: John Snow Labs name: xlm_roberta_qa_mrc2reader date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mrc2reader` is a English model originally trained by `Dongjae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_mrc2reader_en_4.0.0_3.0_1655987882933.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_mrc2reader_en_4.0.0_3.0_1655987882933.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_mrc2reader","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_mrc2reader","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_mrc2reader| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Dongjae/mrc2reader --- layout: model title: Legal Dispute Resolve Clause Binary Classifier author: John Snow Labs name: legclf_dispute_resol_clause date: 2023-02-13 tags: [en, legal, classification, dispute, resolve, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `dispute_resol` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `dispute_resol`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_dispute_resol_clause_en_1.0.0_3.0_1676303229805.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_dispute_resol_clause_en_1.0.0_3.0_1676303229805.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_dispute_resol_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[dispute_resol]| |[other]| |[other]| |[dispute_resol]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_dispute_resol_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support dispute_resol 0.94 0.89 0.92 19 other 0.83 0.91 0.87 11 accuracy - - 0.90 30 macro-avg 0.89 0.90 0.89 30 weighted-avg 0.90 0.90 0.90 30 ``` --- layout: model title: RE Pipeline between Problem, Test, and Findings in Reports author: John Snow Labs name: re_test_problem_finding_pipeline date: 2023-06-13 tags: [licensed, clinical, relation_extraction, problem, test, findings, en] task: Relation Extraction language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_test_problem_finding](https://nlp.johnsnowlabs.com/2021/04/19/re_test_problem_finding_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_test_problem_finding_pipeline_en_4.4.4_3.2_1686665114725.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_test_problem_finding_pipeline_en_4.4.4_3.2_1686665114725.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_test_problem_finding_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Targeted biopsy of this lesion for histological correlation should be considered.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_test_problem_finding_pipeline", "en", "clinical/models") pipeline.fullAnnotate("Targeted biopsy of this lesion for histological correlation should be considered.") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.test_problem_finding.pipeline").predict("""Targeted biopsy of this lesion for histological correlation should be considered.""") ```

## Results ```bash Results | index | relations | entity1 | chunk1 | entity2 | chunk2 | |-------|--------------|--------------|---------------------|--------------|---------| | 0 | 1 | PROCEDURE | biopsy | SYMPTOM | lesion | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_test_problem_finding_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: Pipeline to Detect Clinical Entities (jsl_ner_wip_clinical) author: John Snow Labs name: jsl_ner_wip_clinical_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [jsl_ner_wip_clinical](https://nlp.johnsnowlabs.com/2021/03/31/jsl_ner_wip_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_clinical_pipeline_en_4.3.0_3.2_1678875196882.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_clinical_pipeline_en_4.3.0_3.2_1678875196882.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("jsl_ner_wip_clinical_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("jsl_ner_wip_clinical_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_wip_clinical.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.9984 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 1 | | 2 | male | 38 | 41 | Gender | 0.9986 | | 3 | for 2 days | 48 | 57 | Duration | 0.678133 | | 4 | congestion | 62 | 71 | Symptom | 0.9693 | | 5 | mom | 75 | 77 | Gender | 0.7091 | | 6 | yellow | 99 | 104 | Modifier | 0.667 | | 7 | discharge | 106 | 114 | Symptom | 0.3037 | | 8 | nares | 135 | 139 | External_body_part_or_region | 0.89 | | 9 | she | 147 | 149 | Gender | 0.9992 | | 10 | mild | 168 | 171 | Modifier | 0.8106 | | 11 | problems with his breathing while feeding | 173 | 213 | Symptom | 0.500483 | | 12 | perioral cyanosis | 237 | 253 | Symptom | 0.54895 | | 13 | retractions | 258 | 268 | Symptom | 0.9847 | | 14 | One day ago | 272 | 282 | RelativeDate | 0.550167 | | 15 | mom | 285 | 287 | Gender | 0.573 | | 16 | Tylenol | 345 | 351 | Drug_BrandName | 0.9958 | | 17 | Baby | 354 | 357 | Age | 0.9989 | | 18 | decreased p.o. intake | 377 | 397 | Symptom | 0.22495 | | 19 | His | 400 | 402 | Gender | 0.9997 | | 20 | 20 minutes | 439 | 448 | Duration | 0.1453 | | 21 | q.2h. to | 450 | 457 | Frequency | 0.413667 | | 22 | 5 to 10 minutes | 459 | 473 | Duration | 0.152125 | | 23 | his | 488 | 490 | Gender | 0.9987 | | 24 | respiratory congestion | 492 | 513 | VS_Finding | 0.6458 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Ganda asr_wav2vec2_luganda_by_indonesian_nlp TFWav2Vec2ForCTC from indonesian-nlp author: John Snow Labs name: pipeline_asr_wav2vec2_luganda_by_indonesian_nlp date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_luganda_by_indonesian_nlp` is a Ganda model originally trained by indonesian-nlp. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_luganda_by_indonesian_nlp_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_luganda_by_indonesian_nlp_lg_4.2.0_3.0_1664036315040.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_luganda_by_indonesian_nlp_lg_4.2.0_3.0_1664036315040.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_luganda_by_indonesian_nlp', lang = 'lg') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_luganda_by_indonesian_nlp", lang = "lg") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_luganda_by_indonesian_nlp| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|lg| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Limitation Of Liability Clause Binary Classifier author: John Snow Labs name: legclf_limitation_of_liability_clause date: 2022-12-18 tags: [en, legal, classification, licensed, clause, bert, limitation, of, liability, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the limitation-of-liability clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it’s better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `limitation-of-liability`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_limitation_of_liability_clause_en_1.0.0_3.0_1671393635939.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_limitation_of_liability_clause_en_1.0.0_3.0_1671393635939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_limitation_of_liability_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[limitation-of-liability]| |[other]| |[other]| |[limitation-of-liability]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_limitation_of_liability_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support limitation-of-liability 0.93 0.90 0.91 29 other 0.93 0.95 0.94 39 accuracy - - 0.93 68 macro-avg 0.93 0.92 0.92 68 weighted-avg 0.93 0.93 0.93 68 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Irish author: John Snow Labs name: opus_mt_en_ga date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ga, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ga` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ga_xx_2.7.0_2.4_1609170911340.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ga_xx_2.7.0_2.4_1609170911340.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ga| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Introductory Clause Binary Classifier author: John Snow Labs name: legclf_introduction_clause date: 2022-11-17 tags: [introduction, parties, document, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the first introductory clause, where the Document Type and the Parties are mentioned. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `introduction`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_en_1.0.0_3.0_1668680203953.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_en_1.0.0_3.0_1668680203953.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_introduction_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[introduction]| |[other]| |[other]| |[introduction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_introduction_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house, including CUAD dataset. ## Benchmarking ```bash label precision recall f1-score support introduction 1.00 0.98 0.99 99 other 0.99 1.00 0.99 151 accuracy - - 0.99 250 macro-avg 0.99 0.99 0.99 250 weighted-avg 0.99 0.99 0.99 250 ``` --- layout: model title: Legal Indemnifications Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_indemnifications_bert date: 2023-03-05 tags: [en, legal, classification, clauses, indemnifications, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Indemnifications` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Indemnifications`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnifications_bert_en_1.0.0_3.0_1678050524998.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnifications_bert_en_1.0.0_3.0_1678050524998.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_indemnifications_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Indemnifications]| |[Other]| |[Other]| |[Indemnifications]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnifications_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Indemnifications 0.95 0.89 0.92 106 Other 0.92 0.96 0.94 141 accuracy - - 0.93 247 macro-avg 0.93 0.93 0.93 247 weighted-avg 0.93 0.93 0.93 247 ``` --- layout: model title: English image_classifier_vit_base_beans_demo ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_base_beans_demo date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_beans_demo` is a English model originally trained by nateraw. ## Predicted Entities `angular_leaf_spot`, `bean_rust`, `healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_demo_en_4.1.0_3.0_1660168105525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_demo_en_4.1.0_3.0_1660168105525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_beans_demo", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_beans_demo", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_beans_demo| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Sentence Entity Resolver for Snomed Concepts, INT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_findings_int date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to Snomed codes (INT version) using using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts Snomed Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_int_en_3.0.4_3.0_1621189624936.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_int_en_3.0.4_3.0_1621189624936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 266285003| 0.8867|rheumatic myocard...|266285003:::15529...| |chronic renal ins...| 83|109| PROBLEM| 236425005| 0.2470|chronic renal imp...|236425005:::90688...| | COPD| 113|116| PROBLEM| 413839001| 0.0720|chronic lung dise...|413839001:::41384...| | gastritis| 120|128| PROBLEM| 266502003| 0.3240|acute peptic ulce...|266502003:::45560...| | TIA| 136|138| PROBLEM|353101000119105| 0.0727|prostatic intraep...|353101000119105::...| |a non-ST elevatio...| 182|202| PROBLEM| 233843008| 0.2846|silent myocardial...|233843008:::71942...| |Guaiac positive s...| 208|229| PROBLEM| 168319009| 0.1167|stool culture pos...|168319009:::70396...| |cardiac catheteri...| 295|317| TEST| 301095005| 0.2137|cardiac finding::...|301095005:::25090...| | PTCA| 324|327|TREATMENT|842741000000109| 0.0631|occlusion of post...|842741000000109::...| | mid LAD lesion| 332|345| PROBLEM| 449567000| 0.0808|overriding left v...|449567000:::25342...| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_findings_int| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[snomed_int_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on SNOMED (INT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: English DistilBertForQuestionAnswering model (from holtin) Squad2 author: John Snow Labs name: distilbert_qa_base_uncased_holtin_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-holtin-finetuned-squad` is a English model originally trained by `holtin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_holtin_finetuned_squad_en_4.0.0_3.0_1654727128971.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_holtin_finetuned_squad_en_4.0.0_3.0_1654727128971.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_holtin_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_holtin_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_holtin").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_holtin_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/holtin/distilbert-base-uncased-holtin-finetuned-squad --- layout: model title: Legal Annual Bonus Clause Binary Classifier author: John Snow Labs name: legclf_annual_bonus_clause date: 2023-01-29 tags: [en, legal, classification, annual, bonus, clauses, annual_bonus, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `annual-bonus` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `annual-bonus`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_annual_bonus_clause_en_1.0.0_3.0_1675005760165.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_annual_bonus_clause_en_1.0.0_3.0_1675005760165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_annual_bonus_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[annual-bonus]| |[other]| |[other]| |[annual-bonus]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_annual_bonus_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support annual-bonus 1.00 0.94 0.97 33 other 0.95 1.00 0.97 39 accuracy - - 0.97 72 macro-avg 0.98 0.97 0.97 72 weighted-avg 0.97 0.97 0.97 72 ``` --- layout: model title: Pipeline to Detect Clinical Entities (WIP Greedy) author: John Snow Labs name: jsl_ner_wip_greedy_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, wip, biobert, greedy, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [jsl_ner_wip_greedy_biobert](https://nlp.johnsnowlabs.com/2021/07/26/jsl_ner_wip_greedy_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_biobert_pipeline_en_3.4.1_3.0_1647866004113.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_biobert_pipeline_en_3.4.1_3.0_1647866004113.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("jsl_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("jsl_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.greedy_wip_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python legal_pipeline = nlp.PretrainedPipeline("legpipe_alias", "en", "legal/models") text = ["""MUTUAL NON-DISCLOSURE AGREEMENT This Mutual Non-Disclosure Agreement (the “Agreement”) is made on _________ by and between: John Snow Labs, a Delaware corporation, registered at 16192 Coastal Highway, Lewes, Delaware 19958 (“John Snow Labs”), and Acentos, S.L, a Spanish corporation, registered at Gran Via 71, 2º floor (“Company”), (each a “party” and together the “parties”). Recitals: John Snow Labs and Company intend to explore the possibility of a business relationship between each other, whereby each party (“Discloser”) may disclose sensitive information to the other party (“Recipient”). The parties agree as follows:"""] result = legal_pipeline.annotate(text) ```

## Results ```bash ['(“John Snow Labs”)', '(“Company”)', '( “ Discloser ” )', '(“Recipient”)'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legpipe_alias| |Type:|pipeline| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|13.1 KB| ## Included Models - DocumentAssembler - TokenizerModel - ContextualParserModel --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-yoruba-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili_sw_4.1.0_3.0_1659356236533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili_sw_4.1.0_3.0_1659356236533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_yoruba_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-yoruba-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Finnish asr_wav2vec2_xlsr_1b_finnish TFWav2Vec2ForCTC from aapot author: John Snow Labs name: asr_wav2vec2_xlsr_1b_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_1b_finnish` is a Finnish model originally trained by aapot. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_1b_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_fi_4.2.0_3.0_1664018554548.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_fi_4.2.0_3.0_1664018554548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_1b_finnish", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_1b_finnish", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_1b_finnish| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|3.6 GB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from monakth) author: John Snow Labs name: distilbert_qa_base_uncased_fine_tuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distillbert-base-uncased-fine-tuned-squad` is a English model originally trained by `monakth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_fine_tuned_squad_en_4.3.0_3.0_1672774847284.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_fine_tuned_squad_en_4.3.0_3.0_1672774847284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_fine_tuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_fine_tuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_fine_tuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/monakth/distillbert-base-uncased-fine-tuned-squad --- layout: model title: English image_classifier_vit_modeversion1_m6_e4 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_modeversion1_m6_e4 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_modeversion1_m6_e4` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion1_m6_e4_en_4.1.0_3.0_1660171803744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion1_m6_e4_en_4.1.0_3.0_1660171803744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_modeversion1_m6_e4", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_modeversion1_m6_e4", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_modeversion1_m6_e4| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: Malay ALBERT Embeddings (Large) author: John Snow Labs name: albert_embeddings_albert_large_bahasa_cased date: 2022-04-14 tags: [albert, embeddings, ms, open_source] task: Embeddings language: ms edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-large-bahasa-cased` is a Malay model orginally trained by `malay-huggingface`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_large_bahasa_cased_ms_3.4.2_3.0_1649954345847.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_large_bahasa_cased_ms_3.4.2_3.0_1649954345847.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_large_bahasa_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ms| |Size:|68.8 MB| |Case sensitive:|false| ## References - https://huggingface.co/malay-huggingface/albert-large-bahasa-cased - https://github.com/huseinzol05/malay-dataset/tree/master/dumping/clean - https://github.com/huseinzol05/malay-dataset/tree/master/corpus/pile - https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/albert --- layout: model title: Japanese BertForMaskedLM Base Cased model (from cl-tohoku) author: John Snow Labs name: bert_embeddings_base_japanese_v2 date: 2022-12-02 tags: [ja, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ja edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-japanese-v2` is a Japanese model originally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_v2_ja_4.2.4_3.0_1670018249307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_v2_ja_4.2.4_3.0_1670018249307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_v2","ja") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_v2","ja") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_japanese_v2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|417.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-v2 - https://github.com/google-research/bert - https://pypi.org/project/unidic-lite/ - https://github.com/cl-tohoku/bert-japanese/tree/v2.0 - https://taku910.github.io/mecab/ - https://github.com/neologd/mecab-ipadic-neologd - https://github.com/polm/fugashi - https://github.com/polm/unidic-lite - https://www.tensorflow.org/tfrc/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-4` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4_en_4.0.0_3.0_1654191739285.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4_en_4.0.0_3.0_1654191739285.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_64d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|378.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-4 --- layout: model title: Translate Welsh to English Pipeline author: John Snow Labs name: translate_cy_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, cy, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `cy` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_cy_en_xx_2.7.0_2.4_1609689849644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_cy_en_xx_2.7.0_2.4_1609689849644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_cy_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_cy_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.cy.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_cy_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-256-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4_en_4.0.0_3.0_1657184760188.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4_en_4.0.0_3.0_1657184760188.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-256-finetuned-squad-seed-4 --- layout: model title: Legal Duration and termination Clause Binary Classifier author: John Snow Labs name: legclf_duration_and_termination_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `duration-and-termination` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `duration-and-termination` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_duration_and_termination_clause_en_1.0.0_3.2_1660122389421.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_duration_and_termination_clause_en_1.0.0_3.2_1660122389421.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_duration_and_termination_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[duration-and-termination]| |[other]| |[other]| |[duration-and-termination]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_duration_and_termination_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support duration-and-termination 0.93 0.89 0.91 28 other 0.97 0.98 0.98 107 accuracy - - 0.96 135 macro-avg 0.95 0.94 0.94 135 weighted-avg 0.96 0.96 0.96 135 ``` --- layout: model title: Detect Living Species author: John Snow Labs name: ner_living_species date: 2022-06-22 tags: [en, ner, clinical, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_en_3.5.3_3.0_1655888659088.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_en_3.5.3_3.0_1655888659088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------+-------+ |ner_chunk |label | +-----------------------+-------+ |woman |HUMAN | |bacterial |SPECIES| |Fusarium spp |SPECIES| |patient |HUMAN | |species |SPECIES| |Fusarium solani complex|SPECIES| |antifungals |SPECIES| +-----------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.1 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.84 0.96 0.90 2950 B-SPECIES 0.73 0.92 0.81 3129 I-HUMAN 0.69 0.68 0.69 145 I-SPECIES 0.66 0.89 0.76 1166 micro-avg 0.76 0.93 0.83 7390 macro-avg 0.73 0.86 0.79 7390 weighted-avg 0.76 0.93 0.83 7390 ``` --- layout: model title: Pipeline to Detect PHI in text (ner_deid_sd_larg) author: John Snow Labs name: ner_deid_sd_large_pipeline date: 2023-03-13 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_sd_large](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_sd_large_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_pipeline_en_4.3.0_3.2_1678733016225.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_pipeline_en_4.3.0_3.2_1678733016225.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_sd_large_pipeline", "en", "clinical/models") text = '''Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_sd_large_pipeline", "en", "clinical/models") val text = "Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.med_ner_large.pipeline").predict("""Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:------------|-------------:| | 0 | 2093-01-13 | 14 | 23 | DATE | 0.9999 | | 1 | David Hale | 27 | 36 | NAME | 0.90085 | | 2 | Hendrickson Ora | 55 | 69 | NAME | 0.94935 | | 3 | 7194334 | 78 | 84 | ID | 0.9988 | | 4 | 01/13/93 | 93 | 100 | DATE | 0.9913 | | 5 | Oliveira | 110 | 117 | NAME | 0.9924 | | 6 | 25 | 121 | 122 | AGE | 0.987 | | 7 | 2079-11-09 | 150 | 159 | DATE | 0.9952 | | 8 | Cocke County Baptist Hospital | 163 | 191 | LOCATION | 0.795975 | | 9 | 0295 Keats Street | 195 | 211 | LOCATION | 0.741567 | | 10 | 302-786-5227 | 221 | 232 | CONTACT | 0.984 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_sd_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_256 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-256` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_256_zh_4.2.4_3.0_1670325978179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_256_zh_4.2.4_3.0_1670325978179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_256","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_256","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_256| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|39.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-256 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: English RobertaForSequenceClassification Base Cased model (from mrm8488) author: John Snow Labs name: roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression date: 2022-07-13 tags: [en, open_source, roberta, sequence_classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-base-finetuned-suicide-depression` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression_en_4.0.0_3.0_1657715865562.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression_en_4.0.0_3.0_1657715865562.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_sequence_classifier_distilroberta_base_finetuned_suicide_depression| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|309.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/distilroberta-base-finetuned-suicide-depression - https://github.com/ayaanzhaque/SDCNL --- layout: model title: German Electra Embeddings (from deepset) author: John Snow Labs name: electra_embeddings_gelectra_base_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gelectra-base-generator` is a German model orginally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_gelectra_base_generator_de_3.4.4_3.0_1652786833144.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_gelectra_base_generator_de_3.4.4_3.0_1652786833144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_gelectra_base_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|128.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gelectra-base-generator - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in English author: gokhanturer name: Ner_conll2003_100d date: 2022-02-08 tags: [opern_source, ner, glove_100d, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.1.2 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This NER model trained with GloVe 100d word embeddings, annotates text to find features like the names of people , places and organizations. ```python nerdl_model = NerDLModel.pretrained("Ner_conll2003_100d", "en", "@gokhanturer")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ``` ## Predicted Entities `PER`, `LOC`, `ORG`, `MISC` {:.btn-box} [Open in Colab](https://colab.research.google.com/drive/1KtA_K-7_xO0oxQ7DhJtU5RPHtGnMzK8z#scrollTo=BP1iPII8PTdb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/gokhanturer/Ner_conll2003_100d_en_3.1.2_3.0_1644322842689.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/gokhanturer/Ner_conll2003_100d_en_3.1.2_3.0_1644322842689.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Open In Colab JOHNSNOWLABS_LOGO.png Colab Setup In [1]: ! pip install -q pyspark==3.1.2 spark-nlp ! pip install -q spark-nlp-display |████████████████████████████████| 212.4 MB 82 kB/s |████████████████████████████████| 140 kB 61.0 MB/s |████████████████████████████████| 198 kB 73.0 MB/s Building wheel for pyspark (setup.py) ... done |████████████████████████████████| 95 kB 2.0 MB/s |████████████████████████████████| 66 kB 3.5 MB/s In [3]: import sparknlp spark = sparknlp.start(gpu = True) from sparknlp.base import * from sparknlp.annotator import * import pyspark.sql.functions as F from sparknlp.training import CoNLL print("Spark NLP version", sparknlp.version()) print("Apache Spark version:", spark.version) spark Spark NLP version 3.4.0 Apache Spark version: 3.1.2 Out[3]: SparkSession - in-memory SparkContext Spark UI Versionv3.1.2Masterlocal[*]AppNameSpark NLP CONLL Data Prep In [2]: !wget -q https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp/master/src/test/resources/conll2003/eng.train !wget -q https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp/master/src/test/resources/conll2003/eng.testa Train Data In [5]: with open ("eng.train") as f: train_data = f.read() print (train_data[:500]) -DOCSTART- -X- -X- O EU NNP B-NP B-ORG rejects VBZ B-VP O German JJ B-NP B-MISC call NN I-NP O to TO B-VP O boycott VB I-VP O British JJ B-NP B-MISC lamb NN I-NP O . . O O Peter NNP B-NP B-PER Blackburn NNP I-NP I-PER BRUSSELS NNP B-NP B-LOC 1996-08-22 CD I-NP O The DT B-NP O European NNP I-NP B-ORG Commission NNP I-NP I-ORG said VBD B-VP O on IN B-PP O Thursday NNP B-NP O it PRP B-NP O disagreed VBD B-VP O with IN B-PP O German JJ B-NP B-MISC advice NN I-NP O to TO B-PP O consumers NNS B-NP In [6]: train_data = CoNLL().readDataset(spark, 'eng.train') train_data.show(3) +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ | text| document| sentence| token| pos| label| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ |EU rejects German...|[{document, 0, 47...|[{document, 0, 47...|[{token, 0, 1, EU...|[{pos, 0, 1, NNP,...|[{named_entity, 0...| | Peter Blackburn|[{document, 0, 14...|[{document, 0, 14...|[{token, 0, 4, Pe...|[{pos, 0, 4, NNP,...|[{named_entity, 0...| | BRUSSELS 1996-08-22|[{document, 0, 18...|[{document, 0, 18...|[{token, 0, 7, BR...|[{pos, 0, 7, NNP,...|[{named_entity, 0...| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ only showing top 3 rows In [7]: train_data.count() Out[7]: 14041 In [8]: train_data.select(F.explode(F.arrays_zip('token.result', 'pos.result', 'label.result')).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("pos"), F.expr("cols['2']").alias("ner_label")).show(truncate=50) +----------+---+---------+ | token|pos|ner_label| +----------+---+---------+ | EU|NNP| B-ORG| | rejects|VBZ| O| | German| JJ| B-MISC| | call| NN| O| | to| TO| O| | boycott| VB| O| | British| JJ| B-MISC| | lamb| NN| O| | .| .| O| | Peter|NNP| B-PER| | Blackburn|NNP| I-PER| | BRUSSELS|NNP| B-LOC| |1996-08-22| CD| O| | The| DT| O| | European|NNP| B-ORG| |Commission|NNP| I-ORG| | said|VBD| O| | on| IN| O| | Thursday|NNP| O| | it|PRP| O| +----------+---+---------+ only showing top 20 rows In [9]: train_data.select(F.explode(F.arrays_zip("token.result","label.result")).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ground_truth")).groupBy("ground_truth").count().orderBy("count", ascending=False).show(100,truncate=False) +------------+------+ |ground_truth|count | +------------+------+ |O |169578| |B-LOC |7140 | |B-PER |6600 | |B-ORG |6321 | |I-PER |4528 | |I-ORG |3704 | |B-MISC |3438 | |I-LOC |1157 | |I-MISC |1155 | +------------+------+ In [10]: #conll_data.select(F.countDistinct("label.result")).show() #conll_data.groupBy("label.result").count().show(truncate=False) train_data = train_data.withColumn('unique', F.array_distinct("label.result"))\ .withColumn('c', F.size('unique'))\ .filter(F.col('c')>1) train_data.select(F.explode(F.arrays_zip('token.result','label.result')).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ground_truth"))\ .groupBy('ground_truth')\ .count()\ .orderBy('count', ascending=False)\ .show(100,truncate=False) +------------+------+ |ground_truth|count | +------------+------+ |O |137736| |B-LOC |7125 | |B-PER |6596 | |B-ORG |6288 | |I-PER |4528 | |I-ORG |3704 | |B-MISC |3437 | |I-LOC |1157 | |I-MISC |1155 | +------------+------+ Test Data In [11]: with open ("eng.testa") as f: test_data = f.read() print (test_data[:500]) -DOCSTART- -X- -X- O CRICKET NNP B-NP O - : O O LEICESTERSHIRE NNP B-NP B-ORG TAKE NNP I-NP O OVER IN B-PP O AT NNP B-NP O TOP NNP I-NP O AFTER NNP I-NP O INNINGS NNP I-NP O VICTORY NN I-NP O . . O O LONDON NNP B-NP B-LOC 1996-08-30 CD I-NP O West NNP B-NP B-MISC Indian NNP I-NP I-MISC all-rounder NN I-NP O Phil NNP I-NP B-PER Simmons NNP I-NP I-PER took VBD B-VP O four CD B-NP O for IN B-PP O 38 CD B-NP O on IN B-PP O Friday NNP B-NP O as IN B-PP O Leicestershire NNP B-NP B-ORG beat VBD B-VP In [12]: test_data = CoNLL().readDataset(spark, 'eng.testa') test_data.show(3) +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ | text| document| sentence| token| pos| label| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ |CRICKET - LEICEST...|[{document, 0, 64...|[{document, 0, 64...|[{token, 0, 6, CR...|[{pos, 0, 6, NNP,...|[{named_entity, 0...| | LONDON 1996-08-30|[{document, 0, 16...|[{document, 0, 16...|[{token, 0, 5, LO...|[{pos, 0, 5, NNP,...|[{named_entity, 0...| |West Indian all-r...|[{document, 0, 18...|[{document, 0, 18...|[{token, 0, 3, We...|[{pos, 0, 3, NNP,...|[{named_entity, 0...| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ only showing top 3 rows In [13]: test_data.count() Out[13]: 3250 In [14]: test_data.select(F.explode(F.arrays_zip("token.result","label.result")).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ground_truth")).groupBy("ground_truth").count().orderBy("count", ascending=False).show(100,truncate=False) +------------+-----+ |ground_truth|count| +------------+-----+ |O |42759| |B-PER |1842 | |B-LOC |1837 | |B-ORG |1341 | |I-PER |1307 | |B-MISC |922 | |I-ORG |751 | |I-MISC |346 | |I-LOC |257 | +------------+-----+ NERDL Model with Glove_100d In [15]: glove_embeddings = WordEmbeddingsModel.pretrained()\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") glove_100d download started this may take some time. Approximate size to download 145.3 MB [OK!] In [16]: glove_embeddings.transform(test_data).write.parquet('test_data_embeddings.parquet') In [17]: nerTagger = NerDLApproach()\ .setInputCols(["sentence", "token", "embeddings"])\ .setLabelColumn("label")\ .setOutputCol("ner")\ .setMaxEpochs(8)\ .setLr(0.002)\ .setDropout(0.5)\ .setBatchSize(16)\ .setRandomSeed(0)\ .setVerbose(1)\ .setEvaluationLogExtended(True) \ .setEnableOutputLogs(True)\ .setIncludeConfidence(True)\ .setTestDataset('test_data_embeddings.parquet')\ .setEnableMemoryOptimizer(False) ner_pipeline = Pipeline(stages=[ glove_embeddings, nerTagger ]) In [19]: %%time ner_model = ner_pipeline.fit(train_data) CPU times: user 10.6 s, sys: 1.08 s, total: 11.7 s Wall time: 35min 21s In [20]: !cd ~/annotator_logs/ && ls -lt total 16 -rw-r--r-- 1 root root 13178 Feb 6 17:05 NerDLApproach_c5bf4e4c6211.log In [21]: !cat ~/annotator_logs/NerDLApproach_c5bf4e4c6211.log Name of the selected graph: ner-dl/blstm_10_100_128_120.pb Training started - total epochs: 8 - lr: 0.002 - batch size: 16 - labels: 9 - chars: 82 - training examples: 11079 Epoch 1/8 started, lr: 0.002, dataset size: 11079 Epoch 1/8 - 159.93s - loss: 2234.436 - batches: 695 Quality on test dataset: time to finish evaluation: 17.74s label tp fp fn prec rec f1 B-LOC 1695 94 142 0.94745666 0.92270005 0.93491447 I-ORG 528 76 223 0.8741722 0.7030626 0.77933586 I-MISC 255 88 91 0.7434402 0.7369942 0.74020314 I-LOC 189 14 68 0.9310345 0.73540854 0.8217391 I-PER 1270 59 37 0.95560575 0.9716909 0.9635812 B-MISC 797 142 125 0.84877527 0.8644252 0.85652876 B-ORG 1139 170 202 0.8701299 0.8493661 0.85962266 B-PER 1802 176 40 0.91102123 0.9782845 0.94345546 tp: 7675 fp: 819 fn: 928 labels: 8 Macro-average prec: 0.8852045, rec: 0.84524155, f1: 0.86476153 Micro-average prec: 0.903579, rec: 0.8921307, f1: 0.8978184 Epoch 2/8 started, lr: 0.0019900498, dataset size: 11079 Name of the selected graph: ner-dl/blstm_10_100_128_120.pb Training started - total epochs: 8 - lr: 0.002 - batch size: 16 - labels: 9 - chars: 82 - training examples: 11079 Epoch 1/8 started, lr: 0.002, dataset size: 11079 Epoch 2/8 - 246.28s - loss: 839.1736 - batches: 695 Quality on test dataset: time to finish evaluation: 19.66s label tp fp fn prec rec f1 B-LOC 1762 124 75 0.9342524 0.95917255 0.9465484 I-ORG 585 76 166 0.8850227 0.77896136 0.82861185 I-MISC 247 39 99 0.8636364 0.71387285 0.7816456 I-LOC 233 74 24 0.7589577 0.9066148 0.8262412 I-PER 1275 54 32 0.95936793 0.97551644 0.9673748 B-MISC 791 70 131 0.9186992 0.85791755 0.88726866 B-ORG 1150 151 191 0.88393545 0.857569 0.8705526 B-PER 1800 147 42 0.9244992 0.9771987 0.9501188 tp: 7843 fp: 735 fn: 760 labels: 8 Macro-average prec: 0.89104635, rec: 0.8783529, f1: 0.88465416 Micro-average prec: 0.9143157, rec: 0.9116587, f1: 0.9129852 Epoch 3/8 started, lr: 0.001980198, dataset size: 11079 Epoch 1/8 - 254.10s - loss: 2203.116 - batches: 695 Quality on test dataset: time to finish evaluation: 22.30s label tp fp fn prec rec f1 B-LOC 1660 82 177 0.95292765 0.90364724 0.9276334 I-ORG 560 123 191 0.81991214 0.74567246 0.781032 I-MISC 227 65 119 0.7773973 0.65606934 0.7115987 I-LOC 155 10 102 0.93939394 0.6031128 0.73459715 I-PER 1259 60 48 0.954511 0.96327466 0.9588728 B-MISC 762 110 160 0.8738532 0.82646424 0.8494984 B-ORG 1160 237 181 0.83035076 0.8650261 0.8473338 B-PER 1785 170 57 0.9130435 0.96905535 0.94021595 tp: 7568 fp: 857 fn: 1035 labels: 8 Macro-average prec: 0.88267374, rec: 0.81654024, f1: 0.84832007 Micro-average prec: 0.89827895, rec: 0.87969315, f1: 0.8888889 Epoch 2/8 started, lr: 0.0019900498, dataset size: 11079 Epoch 3/8 - 257.88s - loss: 610.81525 - batches: 695 Quality on test dataset: time to finish evaluation: 18.07s label tp fp fn prec rec f1 B-LOC 1764 104 73 0.9443255 0.9602613 0.9522267 I-ORG 640 140 111 0.82051283 0.85219705 0.8360548 I-MISC 227 22 119 0.9116466 0.65606934 0.7630252 I-LOC 223 43 34 0.8383459 0.8677043 0.8527725 I-PER 1265 31 42 0.97608024 0.96786535 0.9719554 B-MISC 785 62 137 0.9268005 0.85141 0.8875071 B-ORG 1207 174 134 0.87400436 0.90007454 0.8868479 B-PER 1795 94 47 0.9502382 0.97448426 0.96220857 tp: 7906 fp: 670 fn: 697 labels: 8 Macro-average prec: 0.90524435, rec: 0.87875825, f1: 0.8918047 Micro-average prec: 0.921875, rec: 0.91898173, f1: 0.9204261 Epoch 4/8 started, lr: 0.0019704434, dataset size: 11079 Epoch 2/8 - 252.19s - loss: 828.8285 - batches: 695 Quality on test dataset: time to finish evaluation: 17.37s label tp fp fn prec rec f1 B-LOC 1722 67 115 0.9625489 0.93739796 0.9498069 I-ORG 624 134 127 0.823219 0.83089215 0.8270378 I-MISC 230 30 116 0.88461536 0.6647399 0.75907594 I-LOC 199 13 58 0.9386792 0.77431905 0.8486141 I-PER 1274 44 33 0.9666161 0.97475135 0.9706667 B-MISC 787 70 135 0.9183197 0.85357916 0.8847667 B-ORG 1212 204 129 0.8559322 0.9038031 0.87921643 B-PER 1807 109 35 0.94311064 0.98099893 0.9616817 tp: 7855 fp: 671 fn: 748 labels: 8 Macro-average prec: 0.9116301, rec: 0.8650602, f1: 0.88773483 Micro-average prec: 0.9212996, rec: 0.9130536, f1: 0.917158 Epoch 3/8 started, lr: 0.001980198, dataset size: 11079 Epoch 4/8 - 250.45s - loss: 512.68085 - batches: 695 Quality on test dataset: time to finish evaluation: 17.78s label tp fp fn prec rec f1 B-LOC 1767 78 70 0.95772356 0.9618944 0.9598045 I-ORG 658 75 93 0.89768076 0.8761651 0.8867924 I-MISC 257 38 89 0.87118644 0.74277455 0.801872 I-LOC 229 18 28 0.9271255 0.8910506 0.9087302 I-PER 1264 21 43 0.9836576 0.9671002 0.97530866 B-MISC 841 127 81 0.86880165 0.9121475 0.8899471 B-ORG 1202 114 139 0.9133739 0.89634603 0.90477985 B-PER 1799 87 43 0.95387065 0.9766558 0.9651288 tp: 8017 fp: 558 fn: 586 labels: 8 Macro-average prec: 0.92167753, rec: 0.9030168, f1: 0.9122517 Micro-average prec: 0.9349271, rec: 0.9318842, f1: 0.93340325 Epoch 5/8 started, lr: 0.0019607844, dataset size: 11079 Epoch 3/8 - 252.61s - loss: 604.5874 - batches: 695 Quality on test dataset: time to finish evaluation: 18.28s label tp fp fn prec rec f1 B-LOC 1764 112 73 0.9402985 0.9602613 0.95017505 I-ORG 614 84 137 0.87965614 0.8175766 0.847481 I-MISC 244 34 102 0.8776978 0.70520234 0.78205127 I-LOC 220 29 37 0.88353413 0.8560311 0.8695652 I-PER 1268 38 39 0.9709035 0.97016066 0.97053194 B-MISC 799 96 123 0.89273745 0.8665944 0.87947166 B-ORG 1205 123 136 0.9073795 0.8985832 0.90295994 B-PER 1792 110 50 0.94216615 0.97285557 0.95726496 tp: 7906 fp: 626 fn: 697 labels: 8 Macro-average prec: 0.9117967, rec: 0.88090813, f1: 0.89608634 Micro-average prec: 0.9266292, rec: 0.91898173, f1: 0.92278963 Epoch 4/8 started, lr: 0.0019704434, dataset size: 11079 Epoch 5/8 - 257.56s - loss: 437.73123 - batches: 695 Quality on test dataset: time to finish evaluation: 17.89s label tp fp fn prec rec f1 B-LOC 1806 163 31 0.91721684 0.9831247 0.94902784 I-ORG 606 26 145 0.95886075 0.8069241 0.8763557 I-MISC 287 99 59 0.7435233 0.82947975 0.78415304 I-LOC 233 54 24 0.8118467 0.9066148 0.85661757 I-PER 1273 26 34 0.9799846 0.9739862 0.9769762 B-MISC 846 146 76 0.8528226 0.9175705 0.8840125 B-ORG 1149 37 192 0.9688027 0.85682327 0.9093787 B-PER 1797 77 45 0.9589114 0.97557 0.96716905 tp: 7997 fp: 628 fn: 606 labels: 8 Macro-average prec: 0.8989962, rec: 0.90626174, f1: 0.9026143 Micro-average prec: 0.9271884, rec: 0.92955947, f1: 0.9283724 Epoch 6/8 started, lr: 0.0019512196, dataset size: 11079 Epoch 4/8 - 255.39s - loss: 508.80334 - batches: 695 Quality on test dataset: time to finish evaluation: 17.63s label tp fp fn prec rec f1 B-LOC 1799 270 38 0.8695022 0.9793141 0.921147 I-ORG 616 97 135 0.86395514 0.82023966 0.8415301 I-MISC 253 33 93 0.88461536 0.73121387 0.8006329 I-LOC 236 117 21 0.66855526 0.91828793 0.77377045 I-PER 1256 18 51 0.98587126 0.96097934 0.9732662 B-MISC 799 66 123 0.92369944 0.8665944 0.89423615 B-ORG 1162 106 179 0.9164038 0.86651754 0.89076275 B-PER 1754 52 88 0.9712071 0.95222586 0.96162283 tp: 7875 fp: 759 fn: 728 labels: 8 Macro-average prec: 0.8854762, rec: 0.8869216, f1: 0.8861983 Micro-average prec: 0.91209173, rec: 0.91537833, f1: 0.9137321 Epoch 5/8 started, lr: 0.0019607844, dataset size: 11079 Epoch 6/8 - 262.11s - loss: 382.8735 - batches: 695 Quality on test dataset: time to finish evaluation: 17.92s label tp fp fn prec rec f1 B-LOC 1749 61 88 0.96629834 0.9520958 0.95914453 I-ORG 682 136 69 0.83374083 0.9081225 0.8693435 I-MISC 268 40 78 0.8701299 0.7745665 0.81957185 I-LOC 215 14 42 0.93886465 0.83657587 0.8847737 I-PER 1280 39 27 0.97043216 0.979342 0.97486675 B-MISC 837 96 85 0.8971061 0.90780914 0.90242594 B-ORG 1232 120 109 0.9112426 0.9187174 0.91496474 B-PER 1795 93 47 0.9507415 0.97448426 0.96246654 tp: 8058 fp: 599 fn: 545 labels: 8 Macro-average prec: 0.91731954, rec: 0.9064642, f1: 0.9118596 Micro-average prec: 0.9308074, rec: 0.93665, f1: 0.9337195 Epoch 7/8 started, lr: 0.0019417477, dataset size: 11079 Epoch 5/8 - 263.75s - loss: 450.50388 - batches: 695 Quality on test dataset: time to finish evaluation: 17.58s label tp fp fn prec rec f1 B-LOC 1749 64 88 0.9646994 0.9520958 0.95835614 I-ORG 689 180 62 0.79286534 0.9174434 0.85061723 I-MISC 275 83 71 0.7681564 0.79479766 0.78124994 I-LOC 210 16 47 0.9292035 0.8171206 0.8695652 I-PER 1271 33 36 0.97469324 0.972456 0.9735733 B-MISC 825 103 97 0.88900864 0.8947939 0.89189196 B-ORG 1239 127 102 0.90702784 0.9239374 0.9154045 B-PER 1791 71 51 0.96186894 0.9723127 0.96706253 tp: 8049 fp: 677 fn: 554 labels: 8 Macro-average prec: 0.89844036, rec: 0.90561974, f1: 0.90201575 Micro-average prec: 0.9224158, rec: 0.93560386, f1: 0.92896307 Epoch 6/8 started, lr: 0.0019512196, dataset size: 11079 Epoch 7/8 - 262.34s - loss: 330.9146 - batches: 695 Quality on test dataset: time to finish evaluation: 18.09s label tp fp fn prec rec f1 B-LOC 1760 74 77 0.95965105 0.9580838 0.9588668 I-ORG 630 36 121 0.9459459 0.8388815 0.88920254 I-MISC 283 93 63 0.75265956 0.8179191 0.7839335 I-LOC 225 20 32 0.9183673 0.8754864 0.8964143 I-PER 1273 32 34 0.97547895 0.9739862 0.974732 B-MISC 837 113 85 0.8810526 0.90780914 0.8942308 B-ORG 1230 96 111 0.9276018 0.91722596 0.92238474 B-PER 1801 70 41 0.9625869 0.9777416 0.97010505 tp: 8039 fp: 534 fn: 564 labels: 8 Macro-average prec: 0.915418, rec: 0.9083917, f1: 0.91189134 Micro-average prec: 0.9377114, rec: 0.93444145, f1: 0.93607366 Epoch 8/8 started, lr: 0.0019323673, dataset size: 11079 Epoch 6/8 - 264.34s - loss: 384.8886 - batches: 695 Quality on test dataset: time to finish evaluation: 18.07s label tp fp fn prec rec f1 B-LOC 1772 84 65 0.95474136 0.96461624 0.9596534 I-ORG 635 50 116 0.9270073 0.8455393 0.88440114 I-MISC 274 78 72 0.77840906 0.7919075 0.7851002 I-LOC 228 19 29 0.9230769 0.8871595 0.9047619 I-PER 1273 32 34 0.97547895 0.9739862 0.974732 B-MISC 842 125 80 0.8707342 0.9132321 0.8914769 B-ORG 1218 86 123 0.93404907 0.9082774 0.920983 B-PER 1791 65 51 0.96497846 0.9723127 0.9686317 tp: 8033 fp: 539 fn: 570 labels: 8 Macro-average prec: 0.9160594, rec: 0.90712893, f1: 0.9115723 Micro-average prec: 0.93712085, rec: 0.933744, f1: 0.9354294 Epoch 7/8 started, lr: 0.0019417477, dataset size: 11079 Epoch 8/8 - 266.21s - loss: 301.41052 - batches: 695 Quality on test dataset: time to finish evaluation: 18.45s label tp fp fn prec rec f1 B-LOC 1768 68 69 0.962963 0.96243876 0.96270084 I-ORG 658 49 93 0.9306931 0.8761651 0.9026063 I-MISC 267 56 79 0.8266254 0.7716763 0.7982063 I-LOC 228 14 29 0.94214875 0.8871595 0.91382766 I-PER 1272 35 35 0.9732211 0.9732211 0.9732211 B-MISC 834 98 88 0.8948498 0.9045553 0.8996764 B-ORG 1239 97 102 0.9273952 0.9239374 0.925663 B-PER 1806 94 36 0.9505263 0.98045605 0.9652592 tp: 8072 fp: 511 fn: 531 labels: 8 Macro-average prec: 0.9260528, rec: 0.90995115, f1: 0.9179313 Micro-average prec: 0.9404637, rec: 0.93827736, f1: 0.9393693 Epoch 7/8 - 256.62s - loss: 335.06775 - batches: 695 Quality on test dataset: time to finish evaluation: 8.79s label tp fp fn prec rec f1 B-LOC 1791 128 46 0.9332986 0.9749592 0.95367414 I-ORG 639 78 112 0.8912134 0.8508655 0.8705722 I-MISC 262 48 84 0.8451613 0.75722545 0.7987805 I-LOC 238 60 19 0.7986577 0.92607003 0.8576577 I-PER 1260 19 47 0.9851446 0.9640398 0.97447795 B-MISC 811 72 111 0.9184598 0.8796095 0.89861494 B-ORG 1215 95 126 0.92748094 0.90604025 0.9166353 B-PER 1786 56 56 0.96959823 0.96959823 0.96959823 tp: 8002 fp: 556 fn: 601 labels: 8 Macro-average prec: 0.90862685, rec: 0.903551, f1: 0.9060818 Micro-average prec: 0.93503153, rec: 0.9301407, f1: 0.9325797 Epoch 8/8 started, lr: 0.0019323673, dataset size: 11079 Epoch 8/8 - 133.22s - loss: 299.64578 - batches: 695 Quality on test dataset: time to finish evaluation: 8.91s label tp fp fn prec rec f1 B-LOC 1746 56 91 0.9689234 0.9504627 0.95960426 I-ORG 673 77 78 0.8973333 0.8961385 0.8967355 I-MISC 270 43 76 0.8626198 0.7803468 0.8194234 I-LOC 223 10 34 0.95708156 0.8677043 0.9102041 I-PER 1272 41 35 0.9687738 0.9732211 0.9709923 B-MISC 832 109 90 0.88416576 0.9023861 0.893183 B-ORG 1264 143 77 0.8983653 0.94258016 0.9199418 B-PER 1801 76 41 0.95950985 0.9777416 0.9685399 tp: 8081 fp: 555 fn: 522 labels: 8 Macro-average prec: 0.9245966, rec: 0.9113227, f1: 0.91791165 Micro-average prec: 0.93573415, rec: 0.9393235, f1: 0.9375254 In [22]: import pyspark.sql.functions as F predictions = ner_model.transform(test_data) predictions.select(F.explode(F.arrays_zip('token.result','label.result','ner.result')).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ground_truth"), F.expr("cols['2']").alias("prediction")).show(truncate=False) +--------------+------------+----------+ |token |ground_truth|prediction| +--------------+------------+----------+ |CRICKET |O |O | |- |O |O | |LEICESTERSHIRE|B-ORG |B-ORG | |TAKE |O |O | |OVER |O |O | |AT |O |O | |TOP |O |O | |AFTER |O |O | |INNINGS |O |O | |VICTORY |O |O | |. |O |O | |LONDON |B-LOC |B-LOC | |1996-08-30 |O |O | |West |B-MISC |B-MISC | |Indian |I-MISC |I-MISC | |all-rounder |O |O | |Phil |B-PER |B-PER | |Simmons |I-PER |I-PER | |took |O |O | |four |O |O | +--------------+------------+----------+ only showing top 20 rows In [23]: from sklearn.metrics import classification_report preds_df = predictions.select(F.explode(F.arrays_zip('token.result','label.result','ner.result')).alias("cols")) \ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ground_truth"), F.expr("cols['2']").alias("prediction")).toPandas() print (classification_report(preds_df['ground_truth'], preds_df['prediction'])) precision recall f1-score support B-LOC 0.97 0.95 0.96 1837 B-MISC 0.88 0.90 0.89 922 B-ORG 0.90 0.94 0.92 1341 B-PER 0.96 0.98 0.97 1842 I-LOC 0.96 0.87 0.91 257 I-MISC 0.86 0.78 0.82 346 I-ORG 0.90 0.90 0.90 751 I-PER 0.97 0.97 0.97 1307 O 1.00 1.00 1.00 42759 accuracy 0.99 51362 macro avg 0.93 0.92 0.93 51362 weighted avg 0.99 0.99 0.99 51362 Saving the Trained Model In [24]: ner_model.stages Out[24]: [WORD_EMBEDDINGS_MODEL_48cffc8b9a76, NerDLModel_6a88a8ead3fd] In [25]: ner_model.stages[1].write().overwrite().save("/content/drive/MyDrive/SparkNLPTask/Ner_glove_100d_e8_b16_lr0.02") Prediction Pipeline In [28]: document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence = SentenceDetector()\ .setInputCols(['document'])\ .setOutputCol('sentence') token = Tokenizer()\ .setInputCols(['sentence'])\ .setOutputCol('token') glove_embeddings = WordEmbeddingsModel.pretrained()\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings") loaded_ner_model = NerDLModel.load("/content/drive/MyDrive/SparkNLPTask/Ner_glove_100d_e8_b16_lr0.02")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_span") ner_prediction_pipeline = Pipeline(stages = [ document, sentence, token, glove_embeddings, loaded_ner_model, converter ]) empty_data = spark.createDataFrame([['']]).toDF("text") prediction_model = ner_prediction_pipeline.fit(empty_data) glove_100d download started this may take some time. Approximate size to download 145.3 MB [OK!] In [33]: text = ''' The final has its own Merseyside subplot, as it will pit Liverpool forwards Mo Salah (of Egypt: pictured above, in white, in the semi-final) and Sadio Mané (of Senegal) against each other. They are just two of the African stars to play for European clubs—the world’s strongest. In fact, only four teams in the English Premier League don’t have a player from the continent. Besides Mr Salah and Mr Mané, Riyad Mahrez of Algeria is at Manchester City, Wilfred Ndidi of Nigeria and Chelsea boasts Edouard Mendy, Senegal’s goalkeeper, and Hakim Ziyech of Morocco. In Italy’s Serie A, Kalidou Koulibaly of Senegal plays for Napoli and Franck Kessie of the Ivory Coast turns out for AC Milan. Eric Maxim Choupo-Moting of Cameroon and Bouna Sarr of Senegal both play for Bayern Munich, the dominant club in Germany’s Bundesliga. ''' sample_data = spark.createDataFrame([[text]]).toDF("text") sample_data.show(truncate=False) +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |text | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | The final has its own Merseyside subplot, as it will pit Liverpool forwards Mo Salah (of Egypt: pictured above, in white, in the semi-final) and Sadio Mané (of Senegal) against each other. They are just two of the African stars to play for European clubs—the world’s strongest. In fact, only four teams in the English Premier League don’t have a player from the continent. Besides Mr Salah and Mr Mané, Riyad Mahrez of Algeria is at Manchester City, Wilfred Ndidi of Nigeria and Chelsea boasts Edouard Mendy, Senegal’s goalkeeper, and Hakim Ziyech of Morocco. In Italy’s Serie A, Kalidou Koulibaly of Senegal plays for Napoli and Franck Kessie of the Ivory Coast turns out for AC Milan. Eric Maxim Choupo-Moting of Cameroon and Bouna Sarr of Senegal both play for Bayern Munich, the dominant club in Germany’s Bundesliga. | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ In [34]: preds = prediction_model.transform(sample_data) result_df = preds.select(F.explode(F.arrays_zip("ner_span.result","ner_span.metadata")).alias("entities")) \ .select(F.expr("entities['0']").alias("chunk"), F.expr("entities['1'].entity").alias("entity")).show(truncate=False) +---------------+------+ |chunk |entity| +---------------+------+ |Merseyside |ORG | |Liverpool |ORG | |Mo Salah |PER | |Egypt |LOC | |Sadio Mané |PER | |Senegal |LOC | |African |MISC | |European |MISC | |English |MISC | |Premier League |ORG | |Mr Salah |PER | |Mr Mané |PER | |Riyad Mahrez |PER | |Algeria |LOC | |Manchester City|LOC | |Wilfred Ndidi |PER | |Nigeria |LOC | |Chelsea |ORG | |Edouard Mendy |PER | |Senegal’s |PER | +---------------+------+ only showing top 20 rows In [35]: from sparknlp.base import LightPipeline light_model = LightPipeline(prediction_model) result = light_model.annotate(text) list(zip(result['token'], result['ner'])) Out[35]: [('The', 'O'), ('final', 'O'), ('has', 'O'), ('its', 'O'), ('own', 'O'), ('Merseyside', 'B-ORG'), ('subplot', 'O'), (',', 'O'), ('as', 'O'), ('it', 'O'), ('will', 'O'), ('pit', 'O'), ('Liverpool', 'B-ORG'), ('forwards', 'O'), ('Mo', 'B-PER'), ('Salah', 'I-PER'), ('(', 'O'), ('of', 'O'), ('Egypt', 'B-LOC'), (':', 'O'), ('pictured', 'O'), ('above', 'O'), (',', 'O'), ('in', 'O'), ('white', 'O'), (',', 'O'), ('in', 'O'), ('the', 'O'), ('semi-final', 'O'), (')', 'O'), ('and', 'O'), ('Sadio', 'B-PER'), ('Mané', 'I-PER'), ('(', 'O'), ('of', 'O'), ('Senegal', 'B-LOC'), (')', 'O'), ('against', 'O'), ('each', 'O'), ('other', 'O'), ('.', 'O'), ('They', 'O'), ('are', 'O'), ('just', 'O'), ('two', 'O'), ('of', 'O'), ('the', 'O'), ('African', 'B-MISC'), ('stars', 'O'), ('to', 'O'), ('play', 'O'), ('for', 'O'), ('European', 'B-MISC'), ('clubs—the', 'O'), ('world’s', 'O'), ('strongest', 'O'), ('.', 'O'), ('In', 'O'), ('fact', 'O'), (',', 'O'), ('only', 'O'), ('four', 'O'), ('teams', 'O'), ('in', 'O'), ('the', 'O'), ('English', 'B-MISC'), ('Premier', 'B-ORG'), ('League', 'I-ORG'), ('don’t', 'O'), ('have', 'O'), ('a', 'O'), ('player', 'O'), ('from', 'O'), ('the', 'O'), ('continent', 'O'), ('.', 'O'), ('Besides', 'O'), ('Mr', 'B-PER'), ('Salah', 'I-PER'), ('and', 'O'), ('Mr', 'B-PER'), ('Mané', 'I-PER'), (',', 'O'), ('Riyad', 'B-PER'), ('Mahrez', 'I-PER'), ('of', 'O'), ('Algeria', 'B-LOC'), ('is', 'O'), ('at', 'O'), ('Manchester', 'B-LOC'), ('City', 'I-LOC'), (',', 'O'), ('Wilfred', 'B-PER'), ('Ndidi', 'I-PER'), ('of', 'O'), ('Nigeria', 'B-LOC'), ('and', 'O'), ('Chelsea', 'B-ORG'), ('boasts', 'O'), ('Edouard', 'B-PER'), ('Mendy', 'I-PER'), (',', 'O'), ('Senegal’s', 'B-PER'), ('goalkeeper', 'O'), (',', 'O'), ('and', 'O'), ('Hakim', 'B-PER'), ('Ziyech', 'I-PER'), ('of', 'O'), ('Morocco', 'B-LOC'), ('.', 'O'), ('In', 'O'), ('Italy’s', 'B-MISC'), ('Serie', 'I-MISC'), ('A', 'I-MISC'), (',', 'O'), ('Kalidou', 'B-PER'), ('Koulibaly', 'I-PER'), ('of', 'O'), ('Senegal', 'B-LOC'), ('plays', 'O'), ('for', 'O'), ('Napoli', 'B-ORG'), ('and', 'O'), ('Franck', 'B-PER'), ('Kessie', 'I-PER'), ('of', 'O'), ('the', 'O'), ('Ivory', 'B-LOC'), ('Coast', 'I-LOC'), ('turns', 'O'), ('out', 'O'), ('for', 'O'), ('AC', 'B-ORG'), ('Milan', 'I-ORG'), ('.', 'O'), ('Eric', 'B-PER'), ('Maxim', 'I-PER'), ('Choupo-Moting', 'I-PER'), ('of', 'O'), ('Cameroon', 'B-LOC'), ('and', 'O'), ('Bouna', 'B-PER'), ('Sarr', 'I-PER'), ('of', 'O'), ('Senegal', 'B-LOC'), ('both', 'O'), ('play', 'O'), ('for', 'O'), ('Bayern', 'B-ORG'), ('Munich', 'I-ORG'), (',', 'O'), ('the', 'O'), ('dominant', 'O'), ('club', 'O'), ('in', 'O'), ('Germany’s', 'B-MISC'), ('Bundesliga', 'I-MISC'), ('.', 'O')] In [37]: import pandas as pd result = light_model.fullAnnotate(text) ner_df= pd.DataFrame([(int(x.metadata['sentence']), x.result, x.begin, x.end, y.result) for x,y in zip(result[0]["token"], result[0]["ner"])], columns=['sent_id','token','start','end','ner']) ner_df.head(15) Out[37]: sent_id token start end ner 0 0 The 1 3 O 1 0 final 5 9 O 2 0 has 11 13 O 3 0 its 15 17 O 4 0 own 19 21 O 5 0 Merseyside 23 32 B-ORG 6 0 subplot 34 40 O 7 0 , 41 41 O 8 0 as 43 44 O 9 0 it 46 47 O 10 0 will 49 52 O 11 0 pit 54 56 O 12 0 Liverpool 58 66 B-ORG 13 0 forwards 68 75 O 14 0 Mo 77 78 B-PER Highlight Entities In [38]: ann_text = light_model.fullAnnotate(text)[0] ann_text.keys() Out[38]: dict_keys(['document', 'ner_span', 'token', 'ner', 'embeddings', 'sentence']) In [39]: from sparknlp_display import NerVisualizer visualiser = NerVisualizer() print ('Standard Output') visualiser.display(ann_text, label_col='ner_span', document_col='document') Standard Output The final has its own Merseyside ORG subplot, as it will pit Liverpool ORG forwards Mo Salah PER (of Egypt LOC: pictured above, in white, in the semi-final) and Sadio Mané PER (of Senegal LOC) against each other. They are just two of the African MISC stars to play for European MISC clubs—the world’s strongest. In fact, only four teams in the English MISC Premier League ORG don’t have a player from the continent. Besides Mr Salah PER and Mr Mané PER, Riyad Mahrez PER of Algeria LOC is at Manchester City LOC, Wilfred Ndidi PER of Nigeria LOC and Chelsea ORG boasts Edouard Mendy PER, Senegal’s PER goalkeeper, and Hakim Ziyech PER of Morocco LOC. In Italy’s Serie A MISC, Kalidou Koulibaly PER of Senegal LOC plays for Napoli ORG and Franck Kessie PER of the Ivory Coast LOC turns out for AC Milan ORG. Eric Maxim Choupo-Moting PER of Cameroon LOC and Bouna Sarr PER of Senegal LOC both play for Bayern Munich ORG, the dominant club in Germany’s Bundesliga MISC. Streamlit In [14]: ! pip install -q pyspark==3.1.2 spark-nlp ! pip install -q spark-nlp-display In [ ]: !pip install streamlit !pip install pyngrok==4.1.1 In [2]: ! wget https://raw.githubusercontent.com/gokhanturer/JSL/main/streamlit_me_ner_model.py --2022-02-06 22:39:33-- https://raw.githubusercontent.com/gokhanturer/JSL/main/streamlit_me_ner_model.py Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ... Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 7979 (7.8K) [text/plain] Saving to: ‘streamlit_me_ner_model.py.3’ streamlit_me_ner_mo 100%[===================>] 7.79K --.-KB/s in 0s 2022-02-06 22:39:34 (93.4 MB/s) - ‘streamlit_me_ner_model.py.3’ saved [7979/7979] In [3]: !ngrok authtoken 24jtZ2Watn1mc1bSG6v19fel7p1_2bYeRjRkniKqqhfgRs6ub Authtoken saved to configuration file: /root/.ngrok2/ngrok.yml In [5]: !streamlit run streamlit_me_ner_model.py &>/dev/null& In [6]: from pyngrok import ngrok public_url = ngrok.connect(port='8501') public_url Out[6]: 'http://2d54-34-125-109-11.ngrok.io' In [7]: !killall ngrok public_url = ngrok.connect(port='8501') public_url Out[7]: 'http://df30-34-125-109-11.ngrok.io' ```

## Results ```bash +---------------+------+ |chunk |entity| +---------------+------+ |Merseyside |ORG | |Liverpool |ORG | |Mo Salah |PER | |Egypt |LOC | |Sadio Mané |PER | |Senegal |LOC | |African |MISC | |European |MISC | |English |MISC | |Premier League |ORG | |Mr Salah |PER | |Mr Mané |PER | |Riyad Mahrez |PER | |Algeria |LOC | |Manchester City|LOC | |Wilfred Ndidi |PER | |Nigeria |LOC | |Chelsea |ORG | |Edouard Mendy |PER | |Senegal’s |PER | +---------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|Ner_conll2003_100d| |Type:|ner| |Compatibility:|Spark NLP 3.1.2+| |License:|Open Source| |Edition:|Community| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|14.3 MB| |Dependencies:|glove100d| ## References This model is trained based on data from : https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp/master/src/test/resources/conll2003/eng.train https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp/master/src/test/resources/conll2003/eng.testa ## Benchmarking ```bash label precision recall f1-score support B-LOC 0.97 0.95 0.96 1837 B-MISC 0.88 0.90 0.89 922 B-ORG 0.90 0.94 0.92 1341 B-PER 0.96 0.98 0.97 1842 I-LOC 0.96 0.87 0.91 257 I-MISC 0.86 0.78 0.82 346 I-ORG 0.90 0.90 0.90 751 I-PER 0.97 0.97 0.97 1307 O 1.00 1.00 1.00 42759 accuracy - - 0.99 51362 macro-avg 0.93 0.92 0.93 51362 weighted-avg 0.99 0.99 0.99 51362 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from akr) author: John Snow Labs name: distilbert_qa_akr_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `akr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_akr_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769755011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_akr_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769755011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_akr_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_akr_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_akr_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/akr/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec2_large_960h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_large_960h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_960h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_en_4.2.0_3.0_1664016568413.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_960h_en_4.2.0_3.0_1664016568413.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_960h", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_960h", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_960h| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|755.4 MB| --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el12 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el12` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el12_en_4.3.0_3.0_1675119407178.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el12_en_4.3.0_3.0_1675119407178.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el12","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el12","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el12| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|184.0 MB| ## References - https://huggingface.co/google/t5-efficient-small-el12 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English BertForQuestionAnswering model (from Sotireas) author: John Snow Labs name: bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext-ContaminationQAmodel_PubmedBERT` is a English model orginally trained by `Sotireas`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT_en_4.0.0_3.0_1654176486544.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT_en_4.0.0_3.0_1654176486544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Sotireas_BiomedNLP_PubMedBERT_base_uncased_abstract_fulltext_ContaminationQAmodel_PubmedBERT| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Sotireas/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext-ContaminationQAmodel_PubmedBERT --- layout: model title: Legal Preamble Clause Binary Classifier author: John Snow Labs name: legclf_preamble_clause date: 2023-02-13 tags: [en, legal, classification, preamble, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `preamble` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `preamble`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_preamble_clause_en_1.0.0_3.0_1676302301456.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_preamble_clause_en_1.0.0_3.0_1676302301456.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_preamble_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[preamble]| |[other]| |[other]| |[preamble]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_preamble_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.88 0.93 16 preamble 0.91 1.00 0.95 21 accuracy - - 0.95 37 macro-avg 0.96 0.94 0.94 37 weighted-avg 0.95 0.95 0.95 37 ``` --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding ICDO Codes author: John Snow Labs name: snomed_icdo_mapping date: 2022-06-27 tags: [snomed, icdo, pipeline, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_icdo_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_icdo_mapping_en_3.5.3_3.0_1656364941154.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_icdo_mapping_en_3.5.3_3.0_1656364941154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("snomed_icdo_mapping", "en", "clinical/models") result= pipeline.fullAnnotate("10376009 2026006 26638004") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("snomed_icdo_mapping", "en", "clinical/models") val result= pipeline.fullAnnotate("10376009 2026006 26638004") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.snomed_to_icdo.pipe").predict("""10376009 2026006 26638004""") ```

## Results ```bash | | snomed_code | icdo_code | |---:|:------------------------------|:-------------------------| | 0 | 10376009 | 2026006 | 26638004 | 8050/2 | 9014/0 | 8322/0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_icdo_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|208.7 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Tagalog RoBERTa Embeddings (Base) author: John Snow Labs name: roberta_embeddings_roberta_tagalog_base date: 2022-04-14 tags: [roberta, embeddings, tl, open_source] task: Embeddings language: tl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-tagalog-base` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_tagalog_base_tl_3.4.2_3.0_1649948855487.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_tagalog_base_tl_3.4.2_3.0_1649948855487.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_tagalog_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|tl| |Size:|410.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/jcblaise/roberta-tagalog-base - https://blaisecruz.com --- layout: model title: Western Frisian BertForMaskedLM Base Cased model (from GroNLP) author: John Snow Labs name: bert_embeddings_base_dutch_cased_frisian date: 2022-12-02 tags: [fy, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: fy edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-dutch-cased-frisian` is a Western Frisian model originally trained by `GroNLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_dutch_cased_frisian_fy_4.2.4_3.0_1670016581644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_dutch_cased_frisian_fy_4.2.4_3.0_1670016581644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_dutch_cased_frisian","fy") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_dutch_cased_frisian","fy") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_dutch_cased_frisian| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fy| |Size:|351.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/GroNLP/bert-base-dutch-cased-frisian - https://arxiv.org/abs/2105.02855 - https://github.com/wietsedv/low-resource-adapt - https://github.com/wietsedv/bertje --- layout: model title: English BertForQuestionAnswering model (from ncduy) author: John Snow Labs name: bert_qa_MiniLM_L12_H384_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `MiniLM-L12-H384-uncased-finetuned-squad` is a English model orginally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_MiniLM_L12_H384_uncased_finetuned_squad_en_4.0.0_3.0_1654178848040.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_MiniLM_L12_H384_uncased_finetuned_squad_en_4.0.0_3.0_1654178848040.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_MiniLM_L12_H384_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_MiniLM_L12_H384_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.mini_lm_base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_MiniLM_L12_H384_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|124.3 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/MiniLM-L12-H384-uncased-finetuned-squad --- layout: model title: ALBERT Large CoNNL-03 NER Pipeline author: John Snow Labs name: albert_large_token_classifier_conll03_pipeline date: 2022-06-19 tags: [open_source, ner, token_classifier, albert, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [albert_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/albert_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653727302.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653727302.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------+-----------+---------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | chunk| entity|icdo_code| all_k_resolutions| all_k_codes| +--------------------------+-----------+---------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | cancers|Oncological| 8000/3|cancer:::carcinoma:::carcinomatosis:::neoplasms:::ceruminous carcinoma::...|8000/3:::8010/3:::8010/9:::800:::8420/3:::8140/3:::8010/3-C76.0:::8010/6...| | urothelial cancer|Oncological| 8120/3|urothelial carcinoma:::urothelial carcinoma in situ of urinary system:::...|8120/3:::8120/2-C68.9:::8010/3-C68.9:::8130/3-C68.9:::8070/3-C68.9:::813...| | malignant melanoma|Oncological| 8720/3|malignant melanoma:::malignant melanoma, of skin:::malignant melanoma, o...|8720/3:::8720/3-C44.9:::8720/3-C06.9:::8720/3-C69.9:::8721/3:::8720/3-C0...| | tumor|Oncological| 8000/1|tumor:::tumorlet:::tumor cells:::askin tumor:::tumor, secondary:::pilar ...|8000/1:::8040/1:::8001/1:::9365/3:::8000/6:::8103/0:::9364/3:::8940/0:::...| |endometroid adenocarcinoma|Oncological| 8380/3|endometrioid adenocarcinoma:::endometrioid adenoma:::scirrhous adenocarc...|8380/3:::8380/0:::8141/3-C54.1:::8560/3-C54.1:::8260/3-C54.1:::8380/3-C5...| | squamous cell carcinoma|Oncological| 8070/3|squamous cell carcinoma:::verrucous squamous cell carcinoma:::squamous c...|8070/3:::8051/3:::8070/2:::8052/3:::8070/3-C44.5:::8075/3:::8560/3:::807...| +--------------------------+-----------+---------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icdo_augmented| |Compatibility:|Healthcare NLP 3.5.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icdo_code]| |Language:|en| |Size:|175.7 MB| |Case sensitive:|false| ## References Trained on ICD-O Histology Behaviour dataset with sbiobert_base_cased_mli sentence embeddings. https://apps.who.int/iris/bitstream/handle/10665/96612/9789241548496_eng.pdf --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_dl12 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-dl12` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl12_en_4.3.0_3.0_1675118545379.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl12_en_4.3.0_3.0_1675118545379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_dl12","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_dl12","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_dl12| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|196.2 MB| ## References - https://huggingface.co/google/t5-efficient-small-dl12 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English image_classifier_vit_blocks ViTForImageClassification from lazyturtl author: John Snow Labs name: image_classifier_vit_blocks date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_blocks` is a English model originally trained by lazyturtl. ## Predicted Entities `red color`, `orange color`, `green color`, `cyan color`, `yellow color`, `blue color` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_blocks_en_4.1.0_3.0_1660166657299.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_blocks_en_4.1.0_3.0_1660166657299.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_blocks", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_blocks", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_blocks| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from Danish to English author: John Snow Labs name: opus_mt_da_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, da, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `da` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_da_en_xx_2.7.0_2.4_1609167272759.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_da_en_xx_2.7.0_2.4_1609167272759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_da_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Indonesian RobertaForMaskedLM Small Cased model (from w11wo) author: John Snow Labs name: roberta_embeddings_indo_small date: 2022-12-12 tags: [id, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: id edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indo-roberta-small` is a Indonesian model originally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indo_small_id_4.2.4_3.0_1670858716049.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indo_small_id_4.2.4_3.0_1670858716049.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indo_small","id") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indo_small","id") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indo_small| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|id| |Size:|313.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/indo-roberta-small - https://arxiv.org/abs/1907.11692 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: English image_classifier_vit_base_patch16_224_recylce_ft ViTForImageClassification from NhatPham author: John Snow Labs name: image_classifier_vit_base_patch16_224_recylce_ft date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_recylce_ft` is a English model originally trained by NhatPham. ## Predicted Entities `Non-Recycle `, `Object`, `Recycle` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_recylce_ft_en_4.1.0_3.0_1660167955114.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_recylce_ft_en_4.1.0_3.0_1660167955114.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_recylce_ft", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_recylce_ft", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_recylce_ft| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Relation Extraction Model Clinical author: John Snow Labs name: re_drug_drug_interaction_clinical class: RelationExtractionModel language: en nav_key: models repository: clinical/models date: 2020-09-03 task: Relation Extraction edition: Healthcare NLP 2.5.5 spark_version: 2.4 tags: [clinical,licensed,relation extraction,en] supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Relation Extraction model based on syntactic features using deep learning. This model can be used to identify drug-drug interactions relationships among drug entities. ## Predicted Entities ``DDI-advise``, ``DDI-effect``, ``DDI-mechanism``, ``DDI-int``, ``DDI-false`` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_drug_drug_interaction_clinical_en_2.5.5_2.4_1599156924424.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_drug_drug_interaction_clinical_en_2.5.5_2.4_1599156924424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use In the table below, `re_drug_drug_interaction_clinical` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:---------------------------------:|-----------------------------------------------------------------------|:------------:|---------------| | re_drug_drug_interaction_clinical | DDI-advise,
DDI-effect,
DDI-mechanism,
DDI-int,
DDI-false | ner_posology | [“drug-drug”] |

{% include programmingLanguageSelectScalaPython.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") words_embedder = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("embeddings") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") ner_tagger = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models")\ .setInputCols("sentences", "tokens", "embeddings")\ .setOutputCol("ner_tags") ner_converter = NerConverter()\ .setInputCols(["sentences", "tokens", "ner_tags"])\ .setOutputCol("ner_chunks") dependency_parser = DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") ddi_re_model = RelationExtractionModel.pretrained("re_drug_drug_interaction_clinical","en","clinical/models")\ .setInputCols("word_embeddings","chunk","pos","dependency")\ .setOutputCol("category") nlp_pipeline = Pipeline(stages=[documenter, sentencer, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_converter, dependency_parser, ddi_re_model]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) annotations = light_pipeline.fullAnnotate("""When carbamazepine is withdrawn from the combination therapy, aripiprazole dose should then be reduced. If additional adrenergic drugs are to be administered by any route, they should be used with caution because the pharmacologically predictable sympathetic effects of Metformin may be potentiated""") ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentencer = new SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val words_embedder = WordEmbeddingsModel().pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val pos_tagger = PerceptronModel().pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val ner_tagger = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val dependency_parser = DependencyParserModel().pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") val ddi_re_model = RelationExtractionModel.pretrained("re_drug_drug_interaction_clinical","en","clinical/models") .setInputCols("word_embeddings","chunk","pos","dependency") .setOutputCol("category") val pipeline = new Pipeline().setStages(Array(documenter, sentencer, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_converter, dependency_parser, ddi_re_model)) val data = Seq("""When carbamazepine is withdrawn from the combination therapy, aripiprazole dose should then be reduced. If additional adrenergic drugs are to be administered by any route, they should be used with caution because the pharmacologically predictable sympathetic effects of Metformin may be potentiated""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash |relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 |entity2_begin | entity2_end | chunk2 | |DDI-advise | DRUG | 5 | 17 | carbamazepine | DRUG | 62 73 | aripiprazole | ``` {:.model-param} ## Model Information {:.table-model} |----------------|-----------------------------------------| | Name: | re_drug_drug_interaction_clinical | | Type: | RelationExtractionModel | | Compatibility: | Spark NLP 2.5.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [word_embeddings, chunk, pos, dependency] | |Output labels: | [category] | | Language: | en | | Case sensitive: | False | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on data gathered and manually annotated by John Snow Labs. {:.h2_title} ## Benchmarking ```bash +-------------+------+------+------+ | relation|recall| prec | f1 | +-------------+------+------+------+ | DDI-effect| 0.76| 0.38 | 0.51 | | DDI-false| 0.72| 0.97 | 0.83 | | DDI-advise| 0.74| 0.39 | 0.51 | +-------------+------+------+------+ ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from comacrae) author: John Snow Labs name: roberta_qa_edav3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-edav3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_edav3_en_4.3.0_3.0_1674220150986.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_edav3_en_4.3.0_3.0_1674220150986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_edav3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_edav3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_edav3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/comacrae/roberta-edav3 --- layout: model title: Multilingual T5ForConditionalGeneration Base Cased model (from Voicelab) author: John Snow Labs name: t5_vlt5_base_keywords date: 2023-01-31 tags: [en, pl, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `vlt5-base-keywords` is a Multilingual model originally trained by `Voicelab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_vlt5_base_keywords_xx_4.3.0_3.0_1675158538277.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_vlt5_base_keywords_xx_4.3.0_3.0_1675158538277.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_vlt5_base_keywords","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_vlt5_base_keywords","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_vlt5_base_keywords| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|1.1 GB| ## References - https://huggingface.co/Voicelab/vlt5-base-keywords - https://nlp-demo-1.voicelab.ai/ - https://arxiv.org/abs/2209.14008 - https://arxiv.org/abs/2209.14008 - https://voicelab.ai/contact/ --- layout: model title: Athena Conditions Entity Resolver (Healthcare) author: John Snow Labs name: chunkresolve_athena_conditions_healthcare class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-09-16 task: Entity Resolution edition: Healthcare NLP 2.6.0 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities Athena Codes and their normalized definition. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_athena_conditions_healthcare_en_2.6.0_2.4_1600265258887.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_athena_conditions_healthcare_en_2.6.0_2.4_1600265258887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use This model requires `embeddings_healthcare_100d` and `ner_healthcare` in the pipeline you use.

{% include programmingLanguageSelectScalaPython.html %} ```python ... athena_re_model = ChunkEntityResolverModel.pretrained("chunkresolve_athena_conditions_healthcare","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_athena = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter, chunk_embeddings, athena_re_model]) model = pipeline_athena.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala val athena_re_model = ChunkEntityResolverModel.pretrained("chunkresolve_athena_conditions_healthcare","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter, chunk_embeddings, athena_re_model)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash chunk entity athena_description athena_code 0 a cold PROBLEM Intolerant of cold 4213725 1 cough PROBLEM Cough 254761 2 runny nose PROBLEM O/E - nose 4156058 3 fever PROBLEM Fever 437663 4 difficulty breathing PROBLEM Difficulty breathing 4041664 5 her cough PROBLEM Does cough 4122567 6 dry PROBLEM Dry eyes 4036620 7 hacky PROBLEM Resolving infantile idiopathic scoliosis 44833868 8 physical exam TEST Physical angioedema 37110554 9 a right TM PROBLEM Tuberculosis of thyroid gland, unspecified 44819346 10 fairly congested PROBLEM Tonsil congested 4116401 11 Amoxil TREATMENT Amoxycillin overdose 4173544 12 Aldex TREATMENT Oral lesion 43530620 13 difficulty breathing PROBLEM Difficulty breathing 4041664 14 more congested PROBLEM Nasal congestion 4195085 15 a temperature TEST Tolerance of ambient temperature - finding 4271383 16 congestion PROBLEM Nasal congestion 4195085 ``` {:.model-param} ## Model Information {:.table-model} |----------------|-------------------------------------------| | Name: | chunkresolve_athena_conditions_healthcare | | Type: | ChunkEntityResolverModel | | Compatibility: | 2.6.0 | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_healthcare_100d | {:.h2_title} ## Data Source Trained on Athena dataset. --- layout: model title: German BertForMaskedLM Base Cased model (from deepset) author: John Snow Labs name: bert_embeddings_g_base date: 2022-12-02 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gbert-base` is a German model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_base_de_4.2.4_3.0_1670022131926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_base_de_4.2.4_3.0_1670022131926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_base","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_base","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_g_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gbert-base - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - https://workablehr.s3.amazonaws.com/uploads/account/logo/476306/logo - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Part of Speech for Norwegian Nynorsk author: John Snow Labs name: pos_ud_nynorsk date: 2021-03-09 tags: [part_of_speech, open_source, norwegian_nynorsk, pos_ud_nynorsk, nn] task: Part of Speech Tagging language: nn edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - DET - NOUN - ADP - PUNCT - CCONJ - PRON - VERB - PROPN - AUX - ADJ - ADV - SCONJ - PART - INTJ - NUM - X - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_nynorsk_nn_3.0.0_3.0_1615292123096.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_nynorsk_nn_3.0.0_3.0_1615292123096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_nynorsk", "nn") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Hello from John Snow Labs!']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_nynorsk", "nn") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hello from John Snow Labs!").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs!""] token_df = nlu.load('nn.pos.ud_nynorsk').predict(text) token_df ```

## Results ```bash token pos 0 Hello PROPN 1 from NOUN 2 John NOUN 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_nynorsk| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|nn| --- layout: model title: Recognize Entities OntoNotes pipeline - BERT Medium author: John Snow Labs name: onto_recognize_entities_bert_medium date: 2021-03-23 tags: [open_source, english, onto_recognize_entities_bert_medium, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The onto_recognize_entities_bert_medium is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_medium_en_3.0.0_3.0_1616477173790.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_medium_en_3.0.0_3.0_1616477173790.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('onto_recognize_entities_bert_medium', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_medium", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.ner.onto.bert.medium').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:-------------------------------------------|:-------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[0.0365490540862083,.,...]] | ['O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'O'] | ['John Snow Labs'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_medium| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: Company Name to IRS (Edgar database) author: John Snow Labs name: finel_edgar_irs date: 2022-08-30 tags: [en, finance, companies, edgar, licensed] task: Entity Resolution language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an Entity Linking / Entity Resolution model, which allows you to retrieve the IRS number of a company given its name, using SEC Edgar database. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/ER_EDGAR_CRUNCHBASE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finel_edgar_irs_en_1.0.0_3.2_1661866402930.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finel_edgar_irs_en_1.0.0_3.2_1661866402930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk") \ .setOutputCol("sentence_embeddings") resolver = finance.SentenceEntityResolverModel.pretrained("finel_edgar_irs", "en", "finance/models")\ .setInputCols(["ner_chunk", "sentence_embeddings"]) \ .setOutputCol("irs_code")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = nlp.Pipeline( stages = [ documentAssembler, embeddings, resolver]) lp = LightPipeline(pipelineModel) lp.fullAnnotate("CONTACT GOLD") ```

## Results ```bash +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ | chunk| code | all_codes| resolutions | all_distances| +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ | CONTACT GOLD | 981369960| [981369960, 271989147, 208531222, 273566922, 270348508] |[981369960, 271989147, 208531222, 273566922, 270348508] | [0.1733, 0.3700, 0.3867, 0.4103, 0.4121] | +--------------+-----------+---------------------------------------------------------+--------------------------------------------------------+-------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finel_edgar_irs| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[company_irs_number]| |Language:|en| |Size:|313.8 MB| |Case sensitive:|false| ## References In-house scrapping and postprocessing of SEC Edgar Database --- layout: model title: Word2Vec Embeddings in Palatine German (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, pfl, open_source] task: Embeddings language: pfl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pfl_3.4.1_3.0_1647451106726.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pfl_3.4.1_3.0_1647451106726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|pfl| |Size:|92.0 MB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Legal Method Of Exercise Clause Binary Classifier author: John Snow Labs name: legclf_method_of_exercise_clause date: 2023-01-27 tags: [en, legal, classification, method, exercise, clauses, method_of_exercise, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `method-of-exercise` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `method-of-exercise`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_method_of_exercise_clause_en_1.0.0_3.0_1674821693129.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_method_of_exercise_clause_en_1.0.0_3.0_1674821693129.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_method_of_exercise_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[method-of-exercise]| |[other]| |[other]| |[method-of-exercise]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_method_of_exercise_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support method-of-exercise 0.97 1.00 0.98 32 other 1.00 0.97 0.99 38 accuracy - - 0.99 70 macro-avg 0.98 0.99 0.99 70 weighted-avg 0.99 0.99 0.99 70 ``` --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1655730608726.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1655730608726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_1024d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|439.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-0 --- layout: model title: Recognize Entities DL Pipeline for Finnish - Medium author: John Snow Labs name: entity_recognizer_md date: 2021-03-22 tags: [open_source, finnish, entity_recognizer_md, pipeline, fi] supported: true task: [Named Entity Recognition, Lemmatization] language: fi edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_fi_3.0.0_3.0_1616456428015.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_fi_3.0.0_3.0_1616456428015.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_md', lang = 'fi') annotations = pipeline.fullAnnotate(""Hei John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_md", lang = "fi") val result = pipeline.fullAnnotate("Hei John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei John Snow Labs! ""] result_df = nlu.load('fi.ner.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-------------------------|:------------------------|:---------------------------------|:-----------------------------|:---------------------------------|:--------------------| | 0 | ['Hei John Snow Labs! '] | ['Hei John Snow Labs!'] | ['Hei', 'John', 'Snow', 'Labs!'] | [[0.1868100017309188,.,...]] | ['O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| --- layout: model title: Drug Reviews Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_drug_reviews_webmd date: 2022-07-28 tags: [en, clinical, licensed, public_health, classifier, sequence_classification, drug, review] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT based](https://github.com/dmis-lab/biobert) classifier that can classify drug reviews from WebMD.com ## Predicted Entities `negative`, `positive` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_CHANGE_DRUG_TREATMENT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_drug_reviews_webmd_en_4.0.0_3.0_1659008484818.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_drug_reviews_webmd_en_4.0.0_3.0_1659008484818.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_drug_reviews_webmd", "en", "clinical/models")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) data = spark.createDataFrame(["While it has worked for me, the sweating and chills especially at night when trying to sleep are very off putting and I am not sure if I will stick with it very much longer. My eyese no longer feel like there is something in them and my mouth is definitely not as dry as before but the side effects are too invasive for my liking.", "I previously used Cheratussin but was now dispensed Guaifenesin AC as a cheaper alternative. This stuff does n t work as good as Cheratussin and taste like cherry flavored sugar water."], StringType()).toDF("text") result = pipeline.fit(data).transform(data) result.select("text", "class.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_drug_reviews_webmd", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val data = Seq(Array("While it has worked for me, the sweating and chills especially at night when trying to sleep are very off putting and I am not sure if I will stick with it very much longer. My eyese no longer feel like there is something in them and my mouth is definitely not as dry as before but the side effects are too invasive for my liking.", "I previously used Cheratussin but was now dispensed Guaifenesin AC as a cheaper alternative. This stuff does n t work as good as Cheratussin and taste like cherry flavored sugar water.")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.drug_reviews").predict("""While it has worked for me, the sweating and chills especially at night when trying to sleep are very off putting and I am not sure if I will stick with it very much longer. My eyese no longer feel like there is something in them and my mouth is definitely not as dry as before but the side effects are too invasive for my liking.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_nolog_SciBert_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_nolog_SciBert_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.scibert.v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_nolog_SciBert_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/peggyhuang/nolog-SciBert-v2 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Filial) author: John Snow Labs name: distilbert_qa_filial_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Filial`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_filial_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768555427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_filial_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768555427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_filial_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_filial_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_filial_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Filial/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Waiver Of Subrogation Clause Binary Classifier author: John Snow Labs name: legclf_waiver_of_subrogation_clause date: 2023-01-29 tags: [en, legal, classification, waiver, subrogation, clauses, waiver_of_subrogation, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `waiver-of-subrogation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `waiver-of-subrogation`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_waiver_of_subrogation_clause_en_1.0.0_3.0_1675005652788.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_waiver_of_subrogation_clause_en_1.0.0_3.0_1675005652788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_waiver_of_subrogation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[waiver-of-subrogation]| |[other]| |[other]| |[waiver-of-subrogation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_waiver_of_subrogation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.97 0.96 39 waiver-of-subrogation 0.97 0.94 0.95 32 accuracy - - 0.96 71 macro-avg 0.96 0.96 0.96 71 weighted-avg 0.96 0.96 0.96 71 ``` --- layout: model title: Spanish BertForMaskedLM Base Uncased model (from dccuchile) author: John Snow Labs name: bert_embeddings_base_spanish_wwm_uncased date: 2022-12-02 tags: [es, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-uncased` is a Spanish model originally trained by `dccuchile`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_spanish_wwm_uncased_es_4.2.4_3.0_1670018913004.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_spanish_wwm_uncased_es_4.2.4_3.0_1670018913004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_spanish_wwm_uncased","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_spanish_wwm_uncased","es") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_spanish_wwm_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|412.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/dccuchile/bert-base-spanish-wwm-uncased - https://github.com/google-research/bert - https://github.com/josecannete/spanish-corpora - https://github.com/google-research/bert/blob/master/multilingual.md - https://users.dcc.uchile.cl/~jperez/beto/uncased_2M/tensorflow_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/uncased_2M/pytorch_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/cased_2M/tensorflow_weights.tar.gz - https://users.dcc.uchile.cl/~jperez/beto/cased_2M/pytorch_weights.tar.gz - https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827 - https://www.kaggle.com/nltkdata/conll-corpora - https://github.com/gchaperon/beto-benchmarks/blob/master/conll2002/dev_results_beto-cased_conll2002.txt - https://github.com/facebookresearch/MLDoc - https://github.com/gchaperon/beto-benchmarks/blob/master/MLDoc/dev_results_beto-cased_mldoc.txt - https://github.com/gchaperon/beto-benchmarks/blob/master/MLDoc/dev_results_beto-uncased_mldoc.txt - https://github.com/google-research-datasets/paws/tree/master/pawsx - https://github.com/facebookresearch/XNLI - https://colab.research.google.com/drive/1uRwg4UmPgYIqGYY4gW_Nsw9782GFJbPt - https://www.adere.so/ - https://imfd.cl/en/ - https://www.tensorflow.org/tfrc - https://users.dcc.uchile.cl/~jperez/papers/pml4dc2020.pdf - https://github.com/google-research/bert/blob/master/multilingual.md - https://arxiv.org/pdf/1904.09077.pdf - https://arxiv.org/pdf/1906.01502.pdf - https://arxiv.org/abs/1812.10464 - https://arxiv.org/pdf/1901.07291.pdf - https://arxiv.org/pdf/1904.02099.pdf - https://arxiv.org/pdf/1906.01569.pdf - https://arxiv.org/abs/1908.11828 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from V3RX2000) author: John Snow Labs name: xlmroberta_ner_v3rx2000_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `V3RX2000`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_v3rx2000_base_finetuned_panx_de_4.1.0_3.0_1660430662562.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_v3rx2000_base_finetuned_panx_de_4.1.0_3.0_1660430662562.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_v3rx2000_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_v3rx2000_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_v3rx2000_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/V3RX2000/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Pipeline to Summarize Clinical Notes in Laymen Terms author: John Snow Labs name: summarizer_clinical_laymen_pipeline date: 2023-06-06 tags: [licensed, en, clinical, summarization, laymen_terms] task: Summarization language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_laymen](https://nlp.johnsnowlabs.com/2023/05/31/summarizer_clinical_laymen_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_pipeline_en_4.4.1_3.0_1686085843660.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_pipeline_en_4.4.1_3.0_1686085843660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_laymen_pipeline", "en", "clinical/models") text = """ Olivia Smith was seen in my office for evaluation for elective surgical weight loss on October 6, 2008. Olivia Smith is a 34-year-old female with a BMI of 43. She is 5'6" tall and weighs 267 pounds. She is motivated to attempt surgical weight loss because she has been overweight for over 20 years and wants to have more energy and improve her self-image. She is not only affected physically, but also socially by her weight. When she loses weight she always regains it and she always gains back more weight than she has lost. At one time, she lost 100 pounds and gained the weight back within a year. She has tried numerous commercial weight loss programs including Weight Watcher's for four months in 1992 with 15-pound weight loss, RS for two months in 1990 with six-pound weight loss, Slim Fast for six weeks in 2004 with eight-pound weight loss, an exercise program for two months in 2007 with a five-pound weight loss, Atkin's Diet for three months in 2008 with a ten-pound weight loss, and Dexatrim for one month in 2005 with a five-pound weight loss. She has also tried numerous fat reduction or fad diets. She was on Redux for nine months with a 100-pound weight loss. PAST MEDICAL HISTORY: She has a history of hypertension and shortness of breath. PAST SURGICAL HISTORY: Pertinent for cholecystectomy. PSYCHOLOGICAL HISTORY: Negative. SOCIAL HISTORY: She is single. She drinks alcohol once a week. She does not smoke. FAMILY HISTORY: Pertinent for obesity and hypertension. MEDICATIONS: Include Topamax 100 mg twice daily, Zoloft 100 mg twice daily, Abilify 5 mg daily, Motrin 800 mg daily, and a multivitamin. ALLERGIES: She has no known drug allergies. REVIEW OF SYSTEMS: Negative. PHYSICAL EXAM: This is a pleasant female in no acute distress. Alert and oriented x 3. HEENT: Normocephalic, atraumatic. Extraocular muscles intact, nonicteric sclerae. Chest is clear to auscultation bilaterally. Cardiovascular is normal sinus rhythm. Abdomen is obese, soft, nontender and nondistended. Extremities show no edema, clubbing or cyanosis. ASSESSMENT/PLAN: This is a 34-year-old female with a BMI of 43 who is interested in surgical weight via the gastric bypass as opposed to Lap-Band. Olivia Smith will be asking for a letter of medical necessity from Dr. Andrew Johnson. She will also see my nutritionist and social worker and have an upper endoscopy. Once this is completed, we will submit her to her insurance company for approval. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_laymen_pipeline", "en", "clinical/models") val text = """ Olivia Smith was seen in my office for evaluation for elective surgical weight loss on October 6, 2008. Olivia Smith is a 34-year-old female with a BMI of 43. She is 5'6" tall and weighs 267 pounds. She is motivated to attempt surgical weight loss because she has been overweight for over 20 years and wants to have more energy and improve her self-image. She is not only affected physically, but also socially by her weight. When she loses weight she always regains it and she always gains back more weight than she has lost. At one time, she lost 100 pounds and gained the weight back within a year. She has tried numerous commercial weight loss programs including Weight Watcher's for four months in 1992 with 15-pound weight loss, RS for two months in 1990 with six-pound weight loss, Slim Fast for six weeks in 2004 with eight-pound weight loss, an exercise program for two months in 2007 with a five-pound weight loss, Atkin's Diet for three months in 2008 with a ten-pound weight loss, and Dexatrim for one month in 2005 with a five-pound weight loss. She has also tried numerous fat reduction or fad diets. She was on Redux for nine months with a 100-pound weight loss. PAST MEDICAL HISTORY: She has a history of hypertension and shortness of breath. PAST SURGICAL HISTORY: Pertinent for cholecystectomy. PSYCHOLOGICAL HISTORY: Negative. SOCIAL HISTORY: She is single. She drinks alcohol once a week. She does not smoke. FAMILY HISTORY: Pertinent for obesity and hypertension. MEDICATIONS: Include Topamax 100 mg twice daily, Zoloft 100 mg twice daily, Abilify 5 mg daily, Motrin 800 mg daily, and a multivitamin. ALLERGIES: She has no known drug allergies. REVIEW OF SYSTEMS: Negative. PHYSICAL EXAM: This is a pleasant female in no acute distress. Alert and oriented x 3. HEENT: Normocephalic, atraumatic. Extraocular muscles intact, nonicteric sclerae. Chest is clear to auscultation bilaterally. Cardiovascular is normal sinus rhythm. Abdomen is obese, soft, nontender and nondistended. Extremities show no edema, clubbing or cyanosis. ASSESSMENT/PLAN: This is a 34-year-old female with a BMI of 43 who is interested in surgical weight via the gastric bypass as opposed to Lap-Band. Olivia Smith will be asking for a letter of medical necessity from Dr. Andrew Johnson. She will also see my nutritionist and social worker and have an upper endoscopy. Once this is completed, we will submit her to her insurance company for approval. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash This is a clinical note about a 34-year-old woman who is interested in having weight loss surgery. She has been overweight for over 20 years and wants to have more energy and improve her self-image. She has tried many diets and weight loss programs, but has not been successful in keeping the weight off. She has a history of hypertension and shortness of breath, but is not allergic to any medications. She will have an upper endoscopy and will be contacted by a nutritionist and social worker. The plan is to have her weight loss surgery through the gastric bypass, rather than Lap-Band. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_laymen_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|937.2 MB| ## Included Models - DocumentAssembler - MedicalSummarizer --- layout: model title: English BertForQuestionAnswering Mini Cased model (from M-FAC) author: John Snow Labs name: bert_qa_mini_finetuned_squadv2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-mini-finetuned-squadv2` is a English model originally trained by `M-FAC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mini_finetuned_squadv2_en_4.0.0_3.0_1657187810266.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mini_finetuned_squadv2_en_4.0.0_3.0_1657187810266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mini_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mini_finetuned_squadv2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mini_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/M-FAC/bert-mini-finetuned-squadv2 - https://arxiv.org/pdf/2107.03356.pdf - https://github.com/IST-DASLab/M-FAC --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-64-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1657185376585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1657185376585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_64_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-64-finetuned-squad-seed-10 --- layout: model title: Financial Question Answering (RoBerta) author: John Snow Labs name: finqa_roberta date: 2022-08-09 tags: [en, finance, qa, licensed] task: Question Answering language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Financial RoBerta-based Question Answering model, trained on squad-v2, finetuned on proprietary Financial questions and answers. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finqa_roberta_en_1.0.0_3.2_1660054527812.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finqa_roberta_en_1.0.0_3.2_1660054527812.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash `9.4 mn , down from EUR 11.7` ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finqa_roberta| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|248.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References Trained on squad-v2, finetuned on proprietary Financial questions and answers. --- layout: model title: English XlmRoBertaForQuestionAnswering (from tyqiangz) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_finetuned_chaii date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-chaii` is a English model originally trained by `tyqiangz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_finetuned_chaii_en_4.0.0_3.0_1655989602942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_finetuned_chaii_en_4.0.0_3.0_1655989602942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_finetuned_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_finetuned_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_finetuned_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tyqiangz/xlm-roberta-base-finetuned-chaii --- layout: model title: Dutch Named Entity Recognition (from Davlan) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_ner_hrl date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, nl, open_source] task: Named Entity Recognition language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-ner-hrl` is a Dutch model orginally trained by `Davlan`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_ner_hrl_nl_3.4.2_3.0_1652809603204.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_ner_hrl_nl_3.4.2_3.0_1652809603204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_ner_hrl| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|nl| |Size:|855.9 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Davlan/xlm-roberta-base-ner-hrl - https://camel.abudhabi.nyu.edu/anercorp/ - https://www.clips.uantwerpen.be/conll2003/ner/ - https://www.clips.uantwerpen.be/conll2003/ner/ - https://www.clips.uantwerpen.be/conll2002/ner/ - https: --- layout: model title: Recognize Entities DL pipeline for English - BERT author: John Snow Labs name: recognize_entities_bert date: 2021-03-23 tags: [open_source, english, recognize_entities_bert, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The recognize_entities_bert is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/recognize_entities_bert_en_3.0.0_3.0_1616473903583.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/recognize_entities_bert_en_3.0.0_3.0_1616473903583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('recognize_entities_bert', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("recognize_entities_bert", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.ner.bert').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:-------------------------------------------|:-------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[-0.085488274693489,.,...]] | ['O', 'O', 'I-PER', 'I-PER', 'I-ORG', 'O'] | ['John Snow Labs'] || | document | sentence | token | embeddings | ner | entities | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|recognize_entities_bert| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: Fast Neural Machine Translation Model from Kirundi to English author: John Snow Labs name: opus_mt_rn_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, rn, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `rn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_rn_en_xx_2.7.0_2.4_1609167024774.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_rn_en_xx_2.7.0_2.4_1609167024774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_rn_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from juliusco) author: John Snow Labs name: distilbert_qa_juliusco_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `juliusco`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_juliusco_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771577414.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_juliusco_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771577414.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_juliusco_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_juliusco_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_juliusco_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/juliusco/distilbert-base-uncased-finetuned-squad --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265902` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902_en_4.0.0_3.0_1655984889587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902_en_4.0.0_3.0_1655984889587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265902").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265902| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265902 --- layout: model title: Detect Assertion Status (assertion_wip_large) author: John Snow Labs name: jsl_assertion_wip_large date: 2021-01-18 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.7.0 spark_version: 2.4 tags: [clinical, licensed, assertion, en, ner] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The deep neural network architecture for assertion status detection in Spark NLP is based on a BiLSTM framework, and is a modified version of the architecture proposed by Fancellu et.al. (Fancellu, Lopez, and Webber 2016). Its goal is to classify the assertions made on given medical concepts as being present, absent, or possible in the patient, conditionally present in the patient under certain circumstances, hypothetically present in the patient at some future point, and mentioned in the patient report but associated with someone- else (Uzuner et al. 2011). Apart from what we released in other assertion models, an in-house annotations on a curated dataset (6K clinical notes) is used to augment the base assertion dataset (2010 i2b2/VA). {:.h2_title} ## Predicted Entities `present`, `absent`, `possible`, `planned`, `someoneelse`, `past`. {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_assertion_wip_large_en_2.6.5_2.4_1609091911183.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_assertion_wip_large_en_2.6.5_2.4_1609091911183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel, NerConverter, AssertionDLModel.

{% include programmingLanguageSelectScalaPython.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = sparknlp.annotators.Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("jsl_assertion_wip_large", "en", "clinical/models") \ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion]) light_pipeline = LightPipeline(nlpPipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val nerConverter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("jsl_assertion_wip_large", "en", "clinical/models") .setInputCols(Array("sentence", "ner_chunk", "embeddings")) .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results The output is a dataframe with a sentence per row and an ``"assertion"`` column containing all of the assertion labels in the sentence. The assertion column also contains assertion character indices, and other metadata. To get only the entity chunks and assertion labels, without the metadata, select ``"ner_chunk.result"`` and ``"assertion.result"`` from your output dataframe. ```bash +-----------------------------------------+-----+---+----------------------------+-------+-----------+ |chunk |begin|end|ner_label |sent_id|assertion | +-----------------------------------------+-----+---+----------------------------+-------+-----------+ |21-day-old |17 |26 |Age |0 |present | |Caucasian |28 |36 |Race_Ethnicity |0 |present | |male |38 |41 |Gender |0 |someoneelse| |for 2 days |48 |57 |Duration |0 |present | |congestion |62 |71 |Symptom |0 |present | |mom |75 |77 |Gender |0 |someoneelse| |yellow |99 |104|Modifier |0 |present | |discharge |106 |114|Symptom |0 |present | |nares |135 |139|External_body_part_or_region|0 |someoneelse| |she |147 |149|Gender |0 |present | |mild |168 |171|Modifier |0 |present | |problems with his breathing while feeding|173 |213|Symptom |0 |present | |perioral cyanosis |237 |253|Symptom |0 |absent | |retractions |258 |268|Symptom |0 |absent | |One day ago |272 |282|RelativeDate |1 |someoneelse| |mom |285 |287|Gender |1 |someoneelse| |Tylenol |345 |351|Drug_BrandName |1 |someoneelse| |Baby |354 |357|Age |2 |someoneelse| |decreased p.o. intake |377 |397|Symptom |2 |someoneelse| |His |400 |402|Gender |3 |someoneelse| +-----------------------------------------+-----+---+----------------------------+-------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_assertion_wip_large| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence, ner_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|[en]| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ {:.h2_title} ## Benchmarking ```bash label prec rec f1 absent 0.957 0.949 0.953 someoneelse 0.958 0.936 0.947 planned 0.766 0.657 0.707 possible 0.852 0.884 0.868 past 0.894 0.890 0.892 present 0.902 0.917 0.910 Macro-average 0.888 0.872 0.880 Micro-average 0.908 0.908 0.908 ``` --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_nl10 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-nl10` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl10_en_4.3.0_3.0_1675116839759.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl10_en_4.3.0_3.0_1675116839759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_nl10","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_nl10","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_nl10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|689.4 MB| ## References - https://huggingface.co/google/t5-efficient-large-nl10 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Albanian BertForQuestionAnswering model (from vanadhi) author: John Snow Labs name: bert_qa_bert_base_uncased_fiqa_flm_sq_flit date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: sq edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-fiqa-flm-sq-flit` is a Albanian model orginally trained by `vanadhi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_fiqa_flm_sq_flit_sq_4.0.0_3.0_1654181262795.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_fiqa_flm_sq_flit_sq_4.0.0_3.0_1654181262795.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_fiqa_flm_sq_flit","sq") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_fiqa_flm_sq_flit","sq") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("sq.answer_question.bert.base_uncased.by_vanadhi").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_fiqa_flm_sq_flit| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|sq| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vanadhi/bert-base-uncased-fiqa-flm-sq-flit - https://drive.google.com/file/d/1BlWaV-qVPfpGyJoWQJU9bXQgWCATgxEP/view --- layout: model title: Spanish T5ForConditionalGeneration Cased model (from JorgeSarry) author: John Snow Labs name: t5_est5base date: 2023-01-30 tags: [es, open_source, t5, tensorflow] task: Text Generation language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `est5base` is a Spanish model originally trained by `JorgeSarry`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_est5base_es_4.3.0_3.0_1675101719578.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_est5base_es_4.3.0_3.0_1675101719578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_est5base","es") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_est5base","es") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_est5base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|es| |Size:|511.9 MB| ## References - https://huggingface.co/JorgeSarry/est5base - https://towardsdatascience.com/how-to-adapt-a-multilingual-t5-model-for-a-single-language-b9f94f3d9c90 --- layout: model title: English BertForQuestionAnswering Large Cased model (from Slavka) author: John Snow Labs name: bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-finetuned-log-parser-winlogbeat_nowhitespace_large` is a English model originally trained by `Slavka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg_en_4.0.0_3.0_1657182801327.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg_en_4.0.0_3.0_1657182801327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_cased_finetuned_log_parser_winlogbeat_nowhitespace_larg| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Slavka/bert-base-cased-finetuned-log-parser-winlogbeat_nowhitespace_large --- layout: model title: Japanese BertForMaskedLM Base Cased model (from cl-tohoku) author: John Snow Labs name: bert_embeddings_base_japanese_char_whole_word_masking date: 2022-12-02 tags: [ja, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ja edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-japanese-char-whole-word-masking` is a Japanese model originally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_char_whole_word_masking_ja_4.2.4_3.0_1670018214237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_char_whole_word_masking_ja_4.2.4_3.0_1670018214237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_char_whole_word_masking","ja") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_char_whole_word_masking","ja") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_japanese_char_whole_word_masking| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|334.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking - https://github.com/google-research/bert - https://github.com/cl-tohoku/bert-japanese/tree/v1.0 - https://github.com/attardi/wikiextractor - https://taku910.github.io/mecab/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: English image_classifier_vit_my_bean_VIT ViTForImageClassification from woojinSong author: John Snow Labs name: image_classifier_vit_my_bean_VIT date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_my_bean_VIT` is a English model originally trained by woojinSong. ## Predicted Entities `angular_leaf_spot`, `bean_rust`, `healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_my_bean_VIT_en_4.1.0_3.0_1660167368304.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_my_bean_VIT_en_4.1.0_3.0_1660167368304.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_my_bean_VIT", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_my_bean_VIT", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_my_bean_VIT| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Word2Vec Embeddings in Sinhala (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, si, open_source] task: Embeddings language: si edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_si_3.4.1_3.0_1647457358445.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_si_3.4.1_3.0_1647457358445.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_becas","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_becas","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_bne_becas| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|420.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-bne-ROBERTaBECAS --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_doddle124578 TFWav2Vec2ForCTC from doddle124578 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_doddle124578` is a English model originally trained by doddle124578. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578_en_4.2.0_3.0_1664036115160.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578_en_4.2.0_3.0_1664036115160.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_doddle124578| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Extract Granular Anatomical Entities from Oncology Texts author: John Snow Labs name: ner_oncology_anatomy_granular_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, ner, anatomy] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extractions mentions of anatomical entities using granular labels. Definitions of Predicted Entities: - `Direction`: Directional and laterality terms, such as "left", "right", "bilateral", "upper" and "lower". - `Site_Bone`: Anatomical terms that refer to the human skeleton. - `Site_Brain`: Anatomical terms that refer to the central nervous system (including the brain stem and the cerebellum). - `Site_Breast`: Anatomical terms that refer to the breasts. - `Site_Liver`: Anatomical terms that refer to the liver. - `Site_Lung`: Anatomical terms that refer to the lungs. - `Site_Lymph_Node`: Anatomical terms that refer to lymph nodes, excluding adenopathies. - `Site_Other_Body_Part`: Relevant anatomical terms that are not included in the rest of the anatomical entities. ## Predicted Entities `Direction`, `Site_Bone`, `Site_Brain`, `Site_Breast`, `Site_Liver`, `Site_Lung`, `Site_Lymph_Node`, `Site_Other_Body_Part` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_granular_wip_en_4.0.0_3.0_1664584284877.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_granular_wip_en_4.0.0_3.0_1664584284877.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:--------|:------------| | left | Direction | | breast | Site_Breast | | lungs | Site_Lung | | liver | Site_Liver | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_anatomy_granular_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|859.9 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Direction 601.0 150.0 133.0 734.0 0.80 0.82 0.81 Site_Lymph_Node 415.0 31.0 51.0 466.0 0.93 0.89 0.91 Site_Breast 98.0 6.0 20.0 118.0 0.94 0.83 0.88 Site_Other_Body_Part 713.0 277.0 388.0 1101.0 0.72 0.65 0.68 Site_Bone 176.0 30.0 56.0 232.0 0.85 0.76 0.80 Site_Liver 134.0 77.0 36.0 170.0 0.64 0.79 0.70 Site_Lung 337.0 70.0 106.0 443.0 0.83 0.76 0.79 Site_Brain 164.0 58.0 36.0 200.0 0.74 0.82 0.78 macro_avg 2638.0 699.0 826.0 3464.0 0.81 0.79 0.80 micro_avg NaN NaN NaN NaN 0.79 0.76 0.78 ``` --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from edwardjross) author: John Snow Labs name: xlmroberta_ner_edwardjross_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `edwardjross`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_edwardjross_base_finetuned_panx_de_4.1.0_3.0_1660432506605.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_edwardjross_base_finetuned_panx_de_4.1.0_3.0_1660432506605.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_edwardjross_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_edwardjross_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_edwardjross_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/edwardjross/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal Name Clause Binary Classifier author: John Snow Labs name: legclf_name_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `name` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `name` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_name_clause_en_1.0.0_3.2_1660122668905.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_name_clause_en_1.0.0_3.2_1660122668905.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_name_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[name]| |[other]| |[other]| |[name]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_name_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support name 0.98 0.91 0.94 46 other 0.97 0.99 0.98 148 accuracy - - 0.97 194 macro-avg 0.98 0.95 0.96 194 weighted-avg 0.97 0.97 0.97 194 ``` --- layout: model title: Spanish Deberta Embeddings model (from plncmm) author: John Snow Labs name: deberta_embeddings_cowese_base date: 2023-03-12 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, es, tensorflow] task: Embeddings language: es edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mdeberta-cowese-base-es` is a Spanish model originally trained by `plncmm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_cowese_base_es_4.3.1_3.0_1678657528702.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_cowese_base_es_4.3.1_3.0_1678657528702.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_cowese_base","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_cowese_base","es") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_cowese_base| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|1.0 GB| |Case sensitive:|false| ## References https://huggingface.co/plncmm/mdeberta-cowese-base-es --- layout: model title: Fast Neural Machine Translation Model from English to Ga author: John Snow Labs name: opus_mt_en_gaa date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, gaa, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `gaa` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_gaa_xx_2.7.0_2.4_1609169994333.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_gaa_xx_2.7.0_2.4_1609169994333.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_gaa| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English XLMRobertaForTokenClassification Base Cased model (from tner) author: John Snow Labs name: xlmroberta_ner_base_bionlp2004 date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-bionlp2004` is a English model originally trained by `tner`. ## Predicted Entities `protein`, `dna`, `cell line`, `rna`, `cell type` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_bionlp2004_en_4.1.0_3.0_1660426076485.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_bionlp2004_en_4.1.0_3.0_1660426076485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_bionlp2004","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_bionlp2004","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_bionlp2004| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|783.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/tner/xlm-roberta-base-bionlp2004 - https://github.com/asahi417/tner --- layout: model title: Detect Problems, Tests and Treatments (ner_clinical_large) author: John Snow Labs name: ner_clinical_large date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terms. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `PROBLEM`, `TEST`, `TREATMENT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_large_en_3.0.0_3.0_1617206114650.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_large_en_3.0.0_3.0_1617206114650.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes. BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-----------------------------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------------------------+---------+ |the G-protein-activated inwardly rectifying potassium (GIRK|TREATMENT| |the genomicorganization |TREATMENT| |a candidate gene forType II diabetes mellitus |PROBLEM | |byapproximately |TREATMENT| |single nucleotide polymorphisms |TREATMENT| |aVal366Ala substitution |TREATMENT| |an 8 base-pair |TREATMENT| |insertion/deletion |PROBLEM | |Ourexpression studies |TEST | |the transcript in various humantissues |PROBLEM | |fat andskeletal muscle |PROBLEM | |furtherstudies |PROBLEM | |the KCNJ9 protein |TREATMENT| |evaluation |TEST | |Type II diabetes |PROBLEM | |the treatment |TREATMENT| |breast cancer |PROBLEM | |the standard therapy |TREATMENT| |anthracyclines |TREATMENT| |taxanes |TREATMENT| +-----------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_large| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on augmented 2010 i2b2 challenge data with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | I-TREATMENT | 6625 | 1187 | 1329 | 0.848054 | 0.832914 | 0.840416 | | 1 | I-PROBLEM | 15142 | 1976 | 2542 | 0.884566 | 0.856254 | 0.87018 | | 2 | B-PROBLEM | 11005 | 1065 | 1587 | 0.911765 | 0.873968 | 0.892466 | | 3 | I-TEST | 6748 | 923 | 1264 | 0.879677 | 0.842237 | 0.86055 | | 4 | B-TEST | 8196 | 942 | 1029 | 0.896914 | 0.888455 | 0.892665 | | 5 | B-TREATMENT | 8271 | 1265 | 1073 | 0.867345 | 0.885167 | 0.876165 | | 6 | Macro-average | 55987 | 7358 | 8824 | 0.881387 | 0.863166 | 0.872181 | | 7 | Micro-average | 55987 | 7358 | 8824 | 0.883842 | 0.86385 | 0.873732 | ``` --- layout: model title: Legal Undertaking for costs Clause Binary Classifier author: John Snow Labs name: legclf_undertaking_for_costs_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `undertaking-for-costs` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `undertaking-for-costs` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_undertaking_for_costs_clause_en_1.0.0_3.2_1660123148146.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_undertaking_for_costs_clause_en_1.0.0_3.2_1660123148146.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_undertaking_for_costs_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[undertaking-for-costs]| |[other]| |[other]| |[undertaking-for-costs]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_undertaking_for_costs_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.98 0.98 44 undertaking-for-costs 0.91 0.91 0.91 11 accuracy - - 0.96 55 macro-avg 0.94 0.94 0.94 55 weighted-avg 0.96 0.96 0.96 55 ``` --- layout: model title: French CamemBert Embeddings (from aliasdasd) author: John Snow Labs name: camembert_embeddings_aliasdasd_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `aliasdasd`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_aliasdasd_generic_model_fr_3.4.4_3.0_1653987389063.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_aliasdasd_generic_model_fr_3.4.4_3.0_1653987389063.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_aliasdasd_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/aliasdasd/dummy-model --- layout: model title: Pipeline to Summarize Radiology Reports author: John Snow Labs name: summarizer_radiology_pipeline date: 2023-05-29 tags: [licensed, en, clinical, summarization, radiology] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_radiology](https://nlp.johnsnowlabs.com/2023/04/23/summarizer_jsl_radiology_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_radiology_pipeline_en_4.4.2_3.0_1685401622765.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_radiology_pipeline_en_4.4.2_3.0_1685401622765.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_radiology_pipeline", "en", "clinical/models") text = """INDICATIONS: Peripheral vascular disease with claudication. RIGHT: 1. Normal arterial imaging of right lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic. 4. Ankle brachial index is 0.96. LEFT: 1. Normal arterial imaging of left lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic throughout except in posterior tibial artery where it is biphasic. 4. Ankle brachial index is 1.06. IMPRESSION: Normal arterial imaging of both lower lobes. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_radiology_pipeline", "en", "clinical/models") val text = """INDICATIONS: Peripheral vascular disease with claudication. RIGHT: 1. Normal arterial imaging of right lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic. 4. Ankle brachial index is 0.96. LEFT: 1. Normal arterial imaging of left lower extremity. 2. Peak systolic velocity is normal. 3. Arterial waveform is triphasic throughout except in posterior tibial artery where it is biphasic. 4. Ankle brachial index is 1.06. IMPRESSION: Normal arterial imaging of both lower lobes. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash The patient has peripheral vascular disease with claudication. The right lower extremity shows normal arterial imaging, but the peak systolic velocity is normal. The arterial waveform is triphasic throughout, except for the posterior tibial artery, which is biphasic. The ankle brachial index is 0.96. The impression is normal arterial imaging of both lower lobes. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_radiology_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|937.2 MB| ## Included Models - DocumentAssembler - MedicalSummarizer --- layout: model title: Multilingual XLMRobertaForTokenClassification Large Cased model (from oliverguhr) author: John Snow Labs name: xlmroberta_ner_fullstop_punctuation_multilang_larg date: 2022-08-01 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fullstop-punctuation-multilang-large` is a Multilingual model originally trained by `oliverguhr`. ## Predicted Entities `?`, `:`, `,`, `-`, `0`, `.` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_fullstop_punctuation_multilang_larg_xx_4.1.0_3.0_1659353733692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_fullstop_punctuation_multilang_larg_xx_4.1.0_3.0_1659353733692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_fullstop_punctuation_multilang_larg","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_fullstop_punctuation_multilang_larg","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_fullstop_punctuation_multilang_larg| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/oliverguhr/fullstop-punctuation-multilang-large --- layout: model title: French CamemBert Embeddings (from wangst) author: John Snow Labs name: camembert_embeddings_wangst_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `wangst`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_wangst_generic_model_fr_3.4.4_3.0_1653990612908.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_wangst_generic_model_fr_3.4.4_3.0_1653990612908.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_wangst_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/wangst/dummy-model --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6_en_4.0.0_3.0_1655733114451.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6_en_4.0.0_3.0_1655733114451.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_512d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|432.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-6 --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4_en_4.0.0_3.0_1657192672919.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4_en_4.0.0_3.0_1657192672919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|387.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-4 --- layout: model title: Longformer Large NER Pipeline author: John Snow Labs name: longformer_large_token_classifier_conll03_pipeline date: 2022-06-19 tags: [open_source, ner, token_classifier, longformer, conll, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [longformer_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/10/09/longformer_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653984745.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653984745.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("longformer_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I am working at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("longformer_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I am working at John Snow Labs.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_fine_tuned_squad_es_stevemobs","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_fine_tuned_squad_es_stevemobs","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_fine_tuned_squad_es_stevemobs| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/stevemobs/roberta-large-fine-tuned-squad-es --- layout: model title: English asr_wav2vec2_large_xlsr_moroccan TFWav2Vec2ForCTC from othrif author: John Snow Labs name: asr_wav2vec2_large_xlsr_moroccan date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_moroccan` is a English model originally trained by othrif. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_moroccan_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_moroccan_en_4.2.0_3.0_1664097975990.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_moroccan_en_4.2.0_3.0_1664097975990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_moroccan", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_moroccan", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_moroccan| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Fast Neural Machine Translation Model from Umbundu to English author: John Snow Labs name: opus_mt_umb_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, umb, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `umb` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_umb_en_xx_2.7.0_2.4_1609169540084.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_umb_en_xx_2.7.0_2.4_1609169540084.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_umb_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Isoko to English author: John Snow Labs name: opus_mt_iso_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, iso, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `iso` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_iso_en_xx_2.7.0_2.4_1609169073963.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_iso_en_xx_2.7.0_2.4_1609169073963.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_iso_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_models_6 TFWav2Vec2ForCTC from niclas author: John Snow Labs name: pipeline_asr_models_6 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_models_6` is a English model originally trained by niclas. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_models_6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_models_6_en_4.2.0_3.0_1664098783377.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_models_6_en_4.2.0_3.0_1664098783377.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_models_6', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_models_6", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_models_6| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English image_classifier_vit_occupation_prediction ViTForImageClassification from darshanz author: John Snow Labs name: image_classifier_vit_occupation_prediction date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_occupation_prediction` is a English model originally trained by darshanz. ## Predicted Entities `anchor`, `professor`, `doctor`, `farmer`, `athlete` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_occupation_prediction_en_4.1.0_3.0_1660168721579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_occupation_prediction_en_4.1.0_3.0_1660168721579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_occupation_prediction", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_occupation_prediction", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_occupation_prediction| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from comacrae) author: John Snow Labs name: roberta_qa_unaugv3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-unaugv3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_unaugv3_en_4.3.0_3.0_1674222699716.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_unaugv3_en_4.3.0_3.0_1674222699716.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unaugv3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unaugv3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_unaugv3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/comacrae/roberta-unaugv3 --- layout: model title: English BertForQuestionAnswering model (from xraychen) author: John Snow Labs name: bert_qa_squad_baseline date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-baseline` is a English model orginally trained by `xraychen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad_baseline_en_4.0.0_3.0_1654191934724.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad_baseline_en_4.0.0_3.0_1654191934724.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad_baseline","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad_baseline","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base.by_xraychen").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad_baseline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/xraychen/squad-baseline --- layout: model title: English asr_wav2vec2_base_rj_try_5 TFWav2Vec2ForCTC from rjrohit author: John Snow Labs name: asr_wav2vec2_base_rj_try_5 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_rj_try_5` is a English model originally trained by rjrohit. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_rj_try_5_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_rj_try_5_en_4.2.0_3.0_1664102610582.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_rj_try_5_en_4.2.0_3.0_1664102610582.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_rj_try_5", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_rj_try_5", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_rj_try_5| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: ICD10CM Musculoskeletal Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_musculoskeletal_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-28 task: Entity Resolution edition: Healthcare NLP 2.4.5 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_musculoskeletal_clinical_en_2.4.5_2.4_1588103998999.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_musculoskeletal_clinical_en_2.4.5_2.4_1588103998999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... muscu_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_musculoskeletal_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, muscu_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val muscu_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_musculoskeletal_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, muscu_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash chunk entity icd10_muscu_description icd10_muscu_code 0 a cold, cough PROBLEM Postprocedural hemorrhage of a musculoskeletal... M96831 1 runny nose PROBLEM Acquired deformity of nose M950 2 fever PROBLEM Periodic fever syndromes M041 3 difficulty breathing PROBLEM Other dentofacial functional abnormalities M2659 4 her cough PROBLEM Cervicalgia M542 5 physical exam TEST Pathological fracture, unspecified toe(s), seq... M84479S 6 fairly congested PROBLEM Synovial hypertrophy, not elsewhere classified... M67262 7 Amoxil TREATMENT Torticollis M436 8 Aldex TREATMENT Other soft tissue disorders related to use, ov... M7088 9 difficulty breathing PROBLEM Other dentofacial functional abnormalities M2659 10 more congested PROBLEM Pain in unspecified ankle and joints of unspec... M25579 11 trouble sleeping PROBLEM Low back pain M545 12 congestion PROBLEM Progressive systemic sclerosis M340 ``` {:.model-param} ## Model Information {:.table-model} |----------------|-----------------------------------------------| | Name: | chunkresolve_icd10cm_musculoskeletal_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD10CM Dataset Range: M0000-M9979XXS https://www.icd10data.com/ICD10CM/Codes/M00-M99 --- layout: model title: Fast Neural Machine Translation Model from French-Based Creoles And Pidgins to English author: John Snow Labs name: opus_mt_cpf_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, cpf, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `cpf` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_cpf_en_xx_2.7.0_2.4_1609168557473.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_cpf_en_xx_2.7.0_2.4_1609168557473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_cpf_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_huxxx657_roberta_base_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_huxxx657_roberta_base_finetuned_squad_en_4.0.0_3.0_1655734309712.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_huxxx657_roberta_base_finetuned_squad_en_4.0.0_3.0_1655734309712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_huxxx657_roberta_base_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_huxxx657_roberta_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_huxxx657_roberta_base_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-squad --- layout: model title: Extract Clinical Problem Entities (low granularity) from Voice of the Patient Documents (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_problem_reduced_emb_clinical_medium date: 2023-06-07 tags: [licensed, clinical, ner, en, vop, problem] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts clinical problems from the documents transferred from the patient’s own sentences. The taxonomy is reduced (one label for all clinical problems). ## Predicted Entities `Problem`, `HealthStatus`, `Modifier` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_reduced_emb_clinical_medium_en_4.4.3_3.0_1686148297394.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_reduced_emb_clinical_medium_en_4.4.3_3.0_1686148297394.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------------|:------------| | pain | Problem | | fatigue | Problem | | rheumatoid arthritis | Problem | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_problem_reduced_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_medium| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Problem 6190 1000 1020 7210 0.86 0.86 0.86 HealthStatus 92 32 15 107 0.74 0.86 0.80 Modifier 819 221 320 1139 0.79 0.72 0.75 macro_avg 7101 1253 1355 8456 0.80 0.81 0.80 micro_avg 7101 1253 1355 8456 0.85 0.84 0.84 ``` --- layout: model title: Pipeline to Detect Living Species(roberta_embeddings_BR_BERTo) author: John Snow Labs name: ner_living_species_roberta_pipeline date: 2023-03-13 tags: [pt, ner, clinical, licensed, roberta] task: Named Entity Recognition language: pt edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species_roberta](https://nlp.johnsnowlabs.com/2022/06/22/ner_living_species_roberta_pt_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_roberta_pipeline_pt_4.3.0_3.2_1678732150750.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_roberta_pipeline_pt_4.3.0_3.2_1678732150750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_roberta_pipeline", "pt", "clinical/models") text = '''Mulher de 23 anos, de Capinota, Cochabamba, Bolívia. Ela está no nosso país há quatro anos. Frequentou o departamento de emergência obstétrica onde foi encontrada grávida de 37 semanas, com um colo dilatado de 5 cm e membranas rompidas. O obstetra de emergência realizou um teste de estreptococos negativo e solicitou um hemograma, glucose, bioquímica básica, HBV, HCV e serologia da sífilis.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_roberta_pipeline", "pt", "clinical/models") val text = "Mulher de 23 anos, de Capinota, Cochabamba, Bolívia. Ela está no nosso país há quatro anos. Frequentou o departamento de emergência obstétrica onde foi encontrada grávida de 37 semanas, com um colo dilatado de 5 cm e membranas rompidas. O obstetra de emergência realizou um teste de estreptococos negativo e solicitou um hemograma, glucose, bioquímica básica, HBV, HCV e serologia da sífilis." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:--------------|--------:|------:|:------------|-------------:| | 0 | Mulher | 0 | 5 | HUMAN | 0.9975 | | 1 | país | 71 | 74 | HUMAN | 0.8869 | | 2 | grávida | 163 | 169 | HUMAN | 0.9702 | | 3 | estreptococos | 283 | 295 | SPECIES | 0.9211 | | 4 | HBV | 360 | 362 | SPECIES | 0.9911 | | 5 | HCV | 365 | 367 | SPECIES | 0.9858 | | 6 | sífilis | 384 | 390 | SPECIES | 0.8898 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_roberta_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|654.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Detect Clinical Entities (ner_jsl) author: John Snow Labs name: ner_jsl date: 2021-06-24 tags: [ner, licensed, en, clinical] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state-of-the-art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. This model is the official version of jsl_ner_wip_clinical model. Definitions of Predicted Entities: - `Injury_or_Poisoning`: Physical harm or injury caused to the body, including those caused by accidents, falls, or poisoning of a patient or someone else. - `Direction`: All the information relating to the laterality of the internal and external organs. - `Test`: Mentions of laboratory, pathology, and radiological tests. - `Admission_Discharge`: Terms that indicate the admission and/or the discharge of a patient. - `Death_Entity`: Mentions that indicate the death of a patient. - `Relationship_Status`: State of patients romantic or social relationships (e.g. single, married, divorced). - `Duration`: The duration of a medical treatment or medication use. - `Respiration`: Number of breaths per minute. - `Hyperlipidemia`: Terms that indicate hyperlipidemia with relevant subtypes and synonims. - `Birth_Entity`: Mentions that indicate giving birth. - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Labour_Delivery`: Extractions include stages of labor and delivery. - `Family_History_Header`: identifies section headers that correspond to Family History of the patient. - `BMI`: Numeric values and other text information related to Body Mass Index. - `Temperature`: All mentions that refer to body temperature. - `Alcohol`: Terms that indicate alcohol use, abuse or drinking issues of a patient or someone else. - `Kidney_Disease`: Terms that refer to any kidney diseases (includes mentions of modifiers such as "Acute" or "Chronic"). - `Oncological`: All the cancer, tumor or metastasis related extractions mentioned in the document, of the patient or someone else. - `Medical_History_Header`: Identifies section headers that correspond to Past Medical History of a patient. - `Cerebrovascular_Disease`: All terms that refer to cerebrovascular diseases and events. - `Oxygen_Therapy`: Breathing support triggered by patient or entirely or partially by machine (e.g. ventilator, BPAP, CPAP). - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Psychological_Condition`: All the Mental health diagnosis, disorders, conditions or syndromes of a patient or someone else. - `Heart_Disease`: All mentions of acquired, congenital or degenerative heart diseases. - `Employment`: All mentions of patient or provider occupational titles and employment status . - `Obesity`: Terms related to a patient being obese (overweight and BMI are extracted as different labels). - `Disease_Syndrome_Disorder`: All the diseases mentioned in the document, of the patient or someone else (excluding diseases that are extracted with their specific labels, such as "Heart_Disease" etc.). - `Pregnancy`: All terms related to Pregnancy (excluding terms that are extracted with their specific labels, such as "Labour_Delivery" etc.). - `ImagingFindings`: All mentions of radiographic and imagistic findings. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Medical_Device`: All mentions related to medical devices and supplies. - `Race_Ethnicity`: All terms that refer to racial and national origin of sociocultural groups. - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Treatment`: Includes therapeutic and minimally invasive treatment and procedures (invasive treatments or procedures are extracted as "Procedure"). - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_Ingredient`: Active ingredient/s found in drug products. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Diet`: All mentions and information regarding patients dietary habits. - `External_body_part_or_region`: All mentions related to external body parts or organs that can be examined by naked eye. - `LDL`: All mentions related to the lab test and results for LDL (Low Density Lipoprotein). - `VS_Finding`: Qualitative data (e.g. Fever, Cyanosis, Tachycardia) and any other symptoms that refers to vital signs. - `Allergen`: Allergen related extractions mentioned in the document. - `EKG_Findings`: All mentions of EKG readings. - `Imaging_Technique`: All mentions of special radiographic views or special imaging techniques used in radiology. - `Triglycerides`: All mentions terms related to specific lab test for Triglycerides. - `RelativeTime`: Time references that are relative to different times or events (e.g. words such as "approximately", "in the morning"). - `Gender`: Gender-specific nouns and pronouns. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Social_History_Header`: Identifies section headers that correspond to Social History of a patient. - `Substance_Quantity`: All mentions of substance quantity (quantitative information related to illicit/recreational drugs). - `Diabetes`: All terms related to diabetes mellitus. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `Internal_organ_or_component`: All mentions related to internal body parts or organs that can not be examined by naked eye. - `Clinical_Dept`: Terms that indicate the medical and/or surgical departments. - `Form`: Drug and medication forms available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_BrandName`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Strength`: Potency of one unit of drug (or a combination of drugs) the measurement units available are described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Fetus_NewBorn`: All terms related to fetus, infant, new born (excluding terms that are extracted with their specific labels, such as "Labour_Delivery", "Pregnancy" etc.). - `RelativeDate`: Temporal references that are relative to the date of the text or to any other specific date (e.g. "approximately two years ago", "about two days ago"). - `Height`: All mentions related to a patients height. - `Test_Result`: Terms related to all the test results present in the document (clinical tests results are included). - `Sexually_Active_or_Sexual_Orientation`: All terms that are related to sexuality, sexual orientations and sexual activity. - `Frequency`: Frequency of administration for a dose prescribed. - `Time`: Specific time references (hour and/or minutes). - `Weight`: All mentions related to a patients weight. - `Vaccine`: Generic and brand name of vaccines or vaccination procedure. - `Vital_Signs_Header`: Identifies section headers that correspond to Vital Signs of a patient. - `Communicable_Disease`: Includes all mentions of communicable diseases. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Overweight`: Terms related to the patient being overweight (BMI and Obesity is extracted separately). - `Hypertension`: All terms related to Hypertension (quantitative data such as 150/100 is extracted as Blood_Pressure). - `HDL`: Terms related to the lab test for HDL (High Density Lipoprotein). - `Total_Cholesterol`: Terms related to the lab test and results for cholesterol. - `Smoking`: All mentions of smoking status of a patient. - `Date`: Mentions of an exact date, in any format, including day number, month and/or year. ## Predicted Entities `Injury_or_Poisoning`, `Direction`, `Test`, `Admission_Discharge`, `Death_Entity`, `Relationship_Status`, `Duration`, `Respiration`, `Hyperlipidemia`, `Birth_Entity`, `Age`, `Labour_Delivery`, `Family_History_Header`, `BMI`, `Temperature`, `Alcohol`, `Kidney_Disease`, `Oncological`, `Medical_History_Header`, `Cerebrovascular_Disease`, `Oxygen_Therapy`, `O2_Saturation`, `Psychological_Condition`, `Heart_Disease`, `Employment`, `Obesity`, `Disease_Syndrome_Disorder`, `Pregnancy`, `ImagingFindings`, `Procedure`, `Medical_Device`, `Race_Ethnicity`, `Section_Header`, `Symptom`, `Treatment`, `Substance`, `Route`, `Drug_Ingredient`, `Blood_Pressure`, `Diet`, `External_body_part_or_region`, `LDL`, `VS_Finding`, `Allergen`, `EKG_Findings`, `Imaging_Technique`, `Triglycerides`, `RelativeTime`, `Gender`, `Pulse`, `Social_History_Header`, `Substance_Quantity`, `Diabetes`, `Modifier`, `Internal_organ_or_component`, `Clinical_Dept`, `Form`, `Drug_BrandName`, `Strength`, `Fetus_NewBorn`, `RelativeDate`, `Height`, `Test_Result`, `Sexually_Active_or_Sexual_Orientation`, `Frequency`, `Time`, `Weight`, `Vaccine`, `Vital_Signs_Header`, `Communicable_Disease`, `Dosage`, `Overweight`, `Hypertension`, `HDL`, `Total_Cholesterol`, `Smoking`, `Date` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_en_3.1.0_2.4_1624566960534.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_en_3.1.0_2.4_1624566960534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------+----------------------------+ |chunk |ner_label | +-----------------------------------------+----------------------------+ |21-day-old |Age | |Caucasian |Race_Ethnicity | |male |Gender | |for 2 days |Duration | |congestion |Symptom | |mom |Gender | |yellow |Modifier | |discharge |Symptom | |nares |External_body_part_or_region| |she |Gender | |mild |Modifier | |problems with his breathing while feeding|Symptom | |perioral cyanosis |Symptom | |retractions |Symptom | |One day ago |RelativeDate | |mom |Gender | |Tylenol |Drug_BrandName | |Baby |Age | |decreased p.o. intake |Symptom | |His |Gender | +-----------------------------------------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash entity tp fp fn total precision recall f1 VS_Finding 235.0 46.0 43.0 278.0 0.8363 0.8453 0.8408 Direction 3972.0 465.0 458.0 4430.0 0.8952 0.8966 0.8959 Respiration 82.0 4.0 4.0 86.0 0.9535 0.9535 0.9535 Cerebrovascular_D... 93.0 20.0 24.0 117.0 0.823 0.7949 0.8087 Family_History_He... 88.0 6.0 3.0 91.0 0.9362 0.967 0.9514 Heart_Disease 447.0 82.0 119.0 566.0 0.845 0.7898 0.8164 RelativeTime 158.0 80.0 59.0 217.0 0.6639 0.7281 0.6945 Strength 624.0 58.0 53.0 677.0 0.915 0.9217 0.9183 Smoking 121.0 11.0 4.0 125.0 0.9167 0.968 0.9416 Medical_Device 3716.0 491.0 466.0 4182.0 0.8833 0.8886 0.8859 Pulse 136.0 22.0 14.0 150.0 0.8608 0.9067 0.8831 Psychological_Con... 135.0 9.0 29.0 164.0 0.9375 0.8232 0.8766 Overweight 2.0 1.0 0.0 2.0 0.6667 1.0 0.8 Triglycerides 3.0 0.0 2.0 5.0 1.0 0.6 0.75 Obesity 42.0 5.0 6.0 48.0 0.8936 0.875 0.8842 Admission_Discharge 318.0 24.0 11.0 329.0 0.9298 0.9666 0.9478 HDL 3.0 0.0 0.0 3.0 1.0 1.0 1.0 Diabetes 110.0 14.0 8.0 118.0 0.8871 0.9322 0.9091 Section_Header 3740.0 148.0 157.0 3897.0 0.9619 0.9597 0.9608 Age 627.0 75.0 48.0 675.0 0.8932 0.9289 0.9107 O2_Saturation 34.0 14.0 17.0 51.0 0.7083 0.6667 0.6869 Kidney_Disease 96.0 12.0 34.0 130.0 0.8889 0.7385 0.8067 Test 2504.0 545.0 498.0 3002.0 0.8213 0.8341 0.8276 Communicable_Disease 21.0 10.0 6.0 27.0 0.6774 0.7778 0.7241 Hypertension 162.0 5.0 10.0 172.0 0.9701 0.9419 0.9558 External_body_par... 2626.0 356.0 413.0 3039.0 0.8806 0.8641 0.8723 Oxygen_Therapy 81.0 15.0 14.0 95.0 0.8438 0.8526 0.8482 Modifier 2341.0 404.0 539.0 2880.0 0.8528 0.8128 0.8324 Test_Result 1007.0 214.0 255.0 1262.0 0.8247 0.7979 0.8111 BMI 9.0 1.0 0.0 9.0 0.9 1.0 0.9474 Labour_Delivery 57.0 23.0 33.0 90.0 0.7125 0.6333 0.6706 Employment 271.0 59.0 55.0 326.0 0.8212 0.8313 0.8262 Fetus_NewBorn 66.0 33.0 51.0 117.0 0.6667 0.5641 0.6111 Clinical_Dept 923.0 110.0 83.0 1006.0 0.8935 0.9175 0.9053 Time 29.0 13.0 16.0 45.0 0.6905 0.6444 0.6667 Procedure 3185.0 462.0 501.0 3686.0 0.8733 0.8641 0.8687 Diet 36.0 20.0 45.0 81.0 0.6429 0.4444 0.5255 Oncological 459.0 61.0 55.0 514.0 0.8827 0.893 0.8878 LDL 3.0 0.0 3.0 6.0 1.0 0.5 0.6667 Symptom 7104.0 1302.0 1200.0 8304.0 0.8451 0.8555 0.8503 Temperature 116.0 6.0 8.0 124.0 0.9508 0.9355 0.9431 Vital_Signs_Header 215.0 29.0 24.0 239.0 0.8811 0.8996 0.8903 Relationship_Status 49.0 2.0 1.0 50.0 0.9608 0.98 0.9703 Total_Cholesterol 11.0 4.0 5.0 16.0 0.7333 0.6875 0.7097 Blood_Pressure 158.0 18.0 22.0 180.0 0.8977 0.8778 0.8876 Injury_or_Poisoning 579.0 130.0 127.0 706.0 0.8166 0.8201 0.8184 Drug_Ingredient 1716.0 153.0 132.0 1848.0 0.9181 0.9286 0.9233 Treatment 136.0 36.0 60.0 196.0 0.7907 0.6939 0.7391 Pregnancy 123.0 36.0 51.0 174.0 0.7736 0.7069 0.7387 Vaccine 13.0 2.0 6.0 19.0 0.8667 0.6842 0.7647 Disease_Syndrome_... 2981.0 559.0 446.0 3427.0 0.8421 0.8699 0.8557 Height 30.0 10.0 15.0 45.0 0.75 0.6667 0.7059 Frequency 595.0 99.0 138.0 733.0 0.8573 0.8117 0.8339 Route 858.0 76.0 89.0 947.0 0.9186 0.906 0.9123 Duration 351.0 99.0 108.0 459.0 0.78 0.7647 0.7723 Death_Entity 43.0 14.0 5.0 48.0 0.7544 0.8958 0.819 Internal_organ_or... 6477.0 972.0 991.0 7468.0 0.8695 0.8673 0.8684 Alcohol 80.0 18.0 13.0 93.0 0.8163 0.8602 0.8377 Substance_Quantity 6.0 7.0 4.0 10.0 0.4615 0.6 0.5217 Date 498.0 38.0 19.0 517.0 0.9291 0.9632 0.9459 Hyperlipidemia 47.0 3.0 3.0 50.0 0.94 0.94 0.94 Social_History_He... 99.0 7.0 7.0 106.0 0.934 0.934 0.934 Race_Ethnicity 116.0 0.0 0.0 116.0 1.0 1.0 1.0 Imaging_Technique 40.0 18.0 47.0 87.0 0.6897 0.4598 0.5517 Drug_BrandName 859.0 62.0 61.0 920.0 0.9327 0.9337 0.9332 RelativeDate 566.0 124.0 143.0 709.0 0.8203 0.7983 0.8091 Gender 6096.0 80.0 101.0 6197.0 0.987 0.9837 0.9854 Dosage 244.0 31.0 57.0 301.0 0.8873 0.8106 0.8472 Form 234.0 32.0 55.0 289.0 0.8797 0.8097 0.8432 Medical_History_H... 114.0 9.0 10.0 124.0 0.9268 0.9194 0.9231 Birth_Entity 4.0 2.0 3.0 7.0 0.6667 0.5714 0.6154 Substance 59.0 8.0 11.0 70.0 0.8806 0.8429 0.8613 Sexually_Active_o... 5.0 3.0 4.0 9.0 0.625 0.5556 0.5882 Weight 90.0 10.0 21.0 111.0 0.9 0.8108 0.8531 macro - - - - - - 0.8148 micro - - - - - - 0.8788 ``` --- layout: model title: Swahili BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_sw_cased date: 2022-12-02 tags: [sw, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: sw edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-sw-cased` is a Swahili model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_sw_cased_sw_4.2.4_3.0_1670018949309.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_sw_cased_sw_4.2.4_3.0_1670018949309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_sw_cased","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_sw_cased","sw") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_sw_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|sw| |Size:|370.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-sw-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Legal Whereas Clause Binary Classifier author: John Snow Labs name: legclf_cuad_whereas_clause date: 2022-09-20 tags: [en, legal, classification, clauses, whereas, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `whereas` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `whereas` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_whereas_clause_en_1.0.0_3.2_1663693211440.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_whereas_clause_en_1.0.0_3.2_1663693211440.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_whereas_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[whereas]| |[other]| |[other]| |[whereas]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_whereas_clause| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.4 MB| ## References In-house annotations on CUAD dataset ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.94 0.96 67 whereas 0.91 0.98 0.94 41 accuracy - - 0.95 108 macro-avg 0.95 0.96 0.95 108 weighted-avg 0.96 0.95 0.95 108 ``` --- layout: model title: English RobertaForQuestionAnswering (from saattrupdan) author: John Snow Labs name: roberta_qa_icebert_texas_squad_is_saattrupdan date: 2022-06-21 tags: [is, open_source, question_answering, roberta] task: Question Answering language: is edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `icebert-texas-squad-is` is a English model originally trained by `saattrupdan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_texas_squad_is_saattrupdan_is_4.0.0_3.0_1655789280018.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_texas_squad_is_saattrupdan_is_4.0.0_3.0_1655789280018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_icebert_texas_squad_is_saattrupdan","is") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_icebert_texas_squad_is_saattrupdan","is") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("is.answer_question.squad.roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_icebert_texas_squad_is_saattrupdan| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|is| |Size:|455.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saattrupdan/icebert-texas-squad-is --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_fpdm_ft_news date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fpdm_roberta_FT_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_ft_news_en_4.3.0_3.0_1674211000201.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fpdm_ft_news_en_4.3.0_3.0_1674211000201.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_ft_news","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_ft_news","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fpdm_ft_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|458.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/fpdm_roberta_FT_newsqa --- layout: model title: Hindi Named Entity Recognition (from l3cube-pune) author: John Snow Labs name: bert_ner_hing_bert_lid date: 2022-05-09 tags: [bert, ner, token_classification, hi, open_source] task: Named Entity Recognition language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `hing-bert-lid` is a Hindi model orginally trained by `l3cube-pune`. ## Predicted Entities `EN`, `HI` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_hing_bert_lid_hi_3.4.2_3.0_1652097677950.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_hing_bert_lid_hi_3.4.2_3.0_1652097677950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_hing_bert_lid| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|hi| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/l3cube-pune/hing-bert-lid - https://github.com/l3cube-pune/code-mixed-nlp - https://arxiv.org/abs/2204.08398 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4_en_4.3.0_3.0_1674213450529.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4_en_4.3.0_3.0_1674213450529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|439.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-4 --- layout: model title: Fast Neural Machine Translation Model from Central Bikol to Swedish author: John Snow Labs name: opus_mt_bcl_sv date: 2021-06-01 tags: [open_source, seq2seq, translation, bcl, sv, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: bcl target languages: sv {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_sv_xx_3.1.0_2.4_1622552497758.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bcl_sv_xx_3.1.0_2.4_1622552497758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bcl_sv| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English ElectraForQuestionAnswering Large model (from mrm8488) author: John Snow Labs name: electra_qa_large_finetuned_squadv1 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-large-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_large_finetuned_squadv1_en_4.0.0_3.0_1655920852902.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_large_finetuned_squadv1_en_4.0.0_3.0_1655920852902.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_large_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_large_finetuned_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.large.by_mrm8488").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_large_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/electra-large-finetuned-squadv1 --- layout: model title: Multilingual BertForQuestionAnswering model (from horsbug98) author: John Snow Labs name: bert_qa_Part_1_mBERT_Model_E1 date: 2022-06-02 tags: [en, ar, bn, fi, id, ja, sw, ko, ru, te, th, open_source, question_answering, bert, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Part_1_mBERT_Model_E1` is a Multilingual model orginally trained by `horsbug98`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Part_1_mBERT_Model_E1_xx_4.0.0_3.0_1654178889694.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Part_1_mBERT_Model_E1_xx_4.0.0_3.0_1654178889694.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Part_1_mBERT_Model_E1","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Part_1_mBERT_Model_E1","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.tydiqa.multi_lingual_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Part_1_mBERT_Model_E1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/horsbug98/Part_1_mBERT_Model_E1 --- layout: model title: BERT Sentence Embeddings trained on Wikipedia and BooksCorpus and fine-tuned on SST-2 author: John Snow Labs name: sent_bert_wiki_books_sst2 date: 2021-08-31 tags: [en, open_source, sentence_embeddings, wikipedia_dataset, books_corpus_dataset, sst_2_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/wiki_books/1 and fine-tuned on SST-2. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. This model is intended to be used for a variety of English NLP tasks. The pre-training data contains more formal text and the model may not generalize to more colloquial text such as social media or messages. This model is fine-tuned on the SST-2 and is recommended for use in sentiment analysis tasks. The fine-tuning task uses the Stanford Sentiment Treebank (SST-2) dataset to predict the sentiment in a given sentence. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_sst2_en_3.2.0_3.0_1630412133457.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_sst2_en_3.2.0_3.0_1630412133457.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_sst2", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_sst2", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.wiki_books_sst2').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_wiki_books_sst2| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [Wikipedia dataset](https://dumps.wikimedia.org/) [2]: [BooksCorpus dataset](http://yknzhu.wixsite.com/mbweb) [3]: [Stanford Sentiment Treebank (SST-2) dataset](https://nlp.stanford.edu/sentiment/index.html) This Model has been imported from: https://tfhub.dev/google/experts/bert/wiki_books/sst2/2 --- layout: model title: English DistilBertForSequenceClassification Base Uncased model (from mrm8488) author: John Snow Labs name: distilbert_classifier_base_uncased_newspop_student date: 2022-07-20 tags: [open_source, distilbert, sequence_classifier, classification, newspop, en] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Classification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-newspop-student` is a Spanish model originally trained by `mrm8488`. ## Predicted Entities `palestine`, `obama`, `microsoft`, `economy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_classifier_base_uncased_newspop_student_en_4.0.0_3.0_1658326819970.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_classifier_base_uncased_newspop_student_en_4.0.0_3.0_1658326819970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_classifier_base_uncased_newspop_student| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|249.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://huggingface.co/mrm8488/distilbert-base-uncased-newspop-student --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_hier_triplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191205969.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191205969.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_hier_triplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_hier_triplet_epochs_1_shard_1_squad2.0 --- layout: model title: Stopwords Remover for Kyrgyz language (96 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ky, open_source] task: Stop Words Removal language: ky edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ky_3.4.1_3.0_1646673146794.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ky_3.4.1_3.0_1646673146794.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------+ |result | +-----------------------------+ |[Сен, менден, артык, эмессиң]| +-----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ky| |Size:|1.8 KB| --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-luganda-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `PER`, `DATE`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili_sw_4.1.0_3.0_1659354352086.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili_sw_4.1.0_3.0_1659354352086.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_luganda_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-luganda-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-32-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0_en_4.0.0_3.0_1655732398416.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0_en_4.0.0_3.0_1655732398416.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_32d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|417.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-32-finetuned-squad-seed-0 --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_rbtl3 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rbtl3` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbtl3_zh_4.2.4_3.0_1670327142675.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbtl3_zh_4.2.4_3.0_1670327142675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbtl3","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_rbtl3","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbtl3| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|228.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbtl3 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English T5ForConditionalGeneration Cased model (from gokceuludogan) author: John Snow Labs name: t5_t2t_adex_prompt date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t2t-adeX-prompt` is a English model originally trained by `gokceuludogan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_t2t_adex_prompt_en_4.3.0_3.0_1675107607187.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_t2t_adex_prompt_en_4.3.0_3.0_1675107607187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_t2t_adex_prompt","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_t2t_adex_prompt","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_t2t_adex_prompt| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|925.3 MB| ## References - https://huggingface.co/gokceuludogan/t2t-adeX-prompt - https://github.com/gokceuludogan/boun-tabi-smm4h22 --- layout: model title: Detect PHI for Deidentification purposes (Spanish, augmented) author: John Snow Labs name: ner_deid_subentity_augmented date: 2022-02-16 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities, which is more than the previously released `ner_deid_subentity` model. This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf and MeddoCan datasets, and includes several data augmentation mechanisms. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `ID`, `STREET`, `USERNAME`, `SEX`, `EMAIL`, `ZIP`, `MEDICALRECORD`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_es_3.3.4_3.0_1645006642756.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_es_3.3.4_3.0_1645006642756.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("word_embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_subentity_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") nlpPipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] data = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_deid_subentity_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val data = Seq(text).toDF("text") val results = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.subentity_augmented").predict(""" Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-PATIENT| | Miguel| I-PATIENT| | Martínez| I-PATIENT| | ,| O| | varón| B-SEX| | de| O| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-CITY| | ,| O| | España| B-COUNTRY| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19| O| | el| O| | dia| O| | 14| B-DATE| | de| I-DATE| | Marzo| I-DATE| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| O| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-HOSPITAL| | San| I-HOSPITAL| | Carlos| I-HOSPITAL| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|15.0 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 2022.0 224.0 140.0 2162.0 0.9003 0.9352 0.9174 HOSPITAL 259.0 35.0 50.0 309.0 0.881 0.8382 0.859 DATE 1023.0 12.0 12.0 1035.0 0.9884 0.9884 0.9884 ORGANIZATION 2624.0 516.0 544.0 3168.0 0.8357 0.8283 0.832 CITY 1561.0 339.0 266.0 1827.0 0.8216 0.8544 0.8377 ID 36.0 1.0 3.0 39.0 0.973 0.9231 0.9474 STREET 197.0 14.0 9.0 206.0 0.9336 0.9563 0.9448 USERNAME 10.0 6.0 1.0 11.0 0.625 0.9091 0.7407 SEX 682.0 13.0 11.0 693.0 0.9813 0.9841 0.9827 EMAIL 134.0 0.0 1.0 135.0 1.0 0.9926 0.9963 ZIP 141.0 2.0 1.0 142.0 0.986 0.993 0.9895 MEDICALRECORD 29.0 5.0 0.0 29.0 0.8529 1.0 0.9206 PROFESSION 252.0 27.0 25.0 277.0 0.9032 0.9097 0.9065 PHONE 51.0 11.0 0.0 51.0 0.8226 1.0 0.9027 COUNTRY 505.0 74.0 82.0 587.0 0.8722 0.8603 0.8662 DOCTOR 444.0 26.0 48.0 492.0 0.9447 0.9024 0.9231 AGE 549.0 15.0 7.0 556.0 0.9734 0.9874 0.9804 macro - - - - - - 0.9138 micro - - - - - - 0.8930 ``` --- layout: model title: Legal Separation Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_separation_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, separation, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_separation_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `separation-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `separation-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_separation_agreement_bert_en_1.0.0_3.0_1669371119187.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_separation_agreement_bert_en_1.0.0_3.0_1669371119187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_separation_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[separation-agreement]| |[other]| |[other]| |[separation-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_separation_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.93 0.95 0.94 82 separation-agreement 0.88 0.83 0.85 35 accuracy - - 0.91 117 macro-avg 0.90 0.89 0.90 117 weighted-avg 0.91 0.91 0.91 117 ``` --- layout: model title: English asr_wav2vec2_base_100h_ngram TFWav2Vec2ForCTC from saahith author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_ngram date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_ngram` is a English model originally trained by saahith. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_ngram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_ngram_en_4.2.0_3.0_1664042368247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_ngram_en_4.2.0_3.0_1664042368247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_ngram', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_ngram", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_ngram| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dl6 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dl6` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl6_en_4.3.0_3.0_1675110178520.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl6_en_4.3.0_3.0_1675110178520.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dl6","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dl6","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dl6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|366.3 MB| ## References - https://huggingface.co/google/t5-efficient-base-dl6 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English DistilBertForQuestionAnswering Cased model (from stevhliu) author: John Snow Labs name: distilbert_qa_my_awesome_model date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `my_awesome_qa_model` is a English model originally trained by `stevhliu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_my_awesome_model_en_4.3.0_3.0_1672775319390.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_my_awesome_model_en_4.3.0_3.0_1672775319390.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_my_awesome_model","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_my_awesome_model","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_my_awesome_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/stevhliu/my_awesome_qa_model --- layout: model title: Clinical Deidentification (Spanish) author: John Snow Labs name: clinical_deidentification date: 2023-06-13 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with sciwiki_300d embeddings and can be used to deidentify PHI information from medical texts in Spanish. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask, fake or obfuscate the following entities: `AGE`, `DATE`, `PROFESSION`, `E-MAIL`, `USERNAME`, `LOCATION`, `DOCTOR`, `HOSPITAL`, `PATIENT`, `URL`, `IP`, `MEDICALRECORD`, `IDNUM`, `ORGANIZATION`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `PLATE`, `SEX` and `IPADDR` ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_4.4.4_3.2_1686663754234.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_4.4.4_3.2_1686663754234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Results Masked with entity labels ------------------------------ Datos del paciente. Nombre: . Apellidos: . NHC: . NASS: 04. Domicilio: , 5 B.. Localidad/ Provincia: . CP: . Datos asistenciales. Fecha de nacimiento: . País: . Edad: años Sexo: . Fecha de Ingreso: . : María Merino Viveros NºCol: . Informe clínico del paciente: de años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Servicio de Endocrinología y Nutrición km 12,500 28905 - () Correo electrónico: Masked with chars ------------------------------ Datos del paciente. Nombre: [**] . Apellidos: [*************]. NHC: [*****]. NASS: ** [******] 04. Domicilio: [*******************], 5 B.. Localidad/ Provincia: [****]. CP: [***]. Datos asistenciales. Fecha de nacimiento: [********]. País: [****]. Edad: ** años Sexo: *. Fecha de Ingreso: [********]. [****]: María Merino Viveros NºCol: ** ** [***]. Informe clínico del paciente: [***] de ** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en [*********] en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: [*************] +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente [****] significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. [******************] [******************************] Servicio de Endocrinología y Nutrición [*****************] km 12,500 28905 [****] - [****] ([****]) Correo electrónico: [********************] Masked with fixed length chars ------------------------------ Datos del paciente. Nombre: **** . Apellidos: ****. NHC: ****. NASS: **** **** 04. Domicilio: ****, 5 B.. Localidad/ Provincia: ****. CP: ****. Datos asistenciales. Fecha de nacimiento: ****. País: ****. Edad: **** años Sexo: ****. Fecha de Ingreso: ****. ****: María Merino Viveros NºCol: **** **** ****. Informe clínico del paciente: **** de **** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en **** en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: **** +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente **** significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. **** **** Servicio de Endocrinología y Nutrición **** km 12,500 28905 **** - **** (****) Correo electrónico: **** Obfuscated ------------------------------ Datos del paciente. Nombre: Sr. Lerma . Apellidos: Aristides Gonzalez Gelabert. NHC: BBBBBBBBQR648597. NASS: 041010000011 RZRM020101906017 04. Domicilio: Valencia, 5 B.. Localidad/ Provincia: Madrid. CP: 99335. Datos asistenciales. Fecha de nacimiento: 25/04/1977. País: Barcelona. Edad: 8 años Sexo: F.. Fecha de Ingreso: 02/08/2018. transportista: María Merino Viveros NºCol: olegario10 olegario10 felisa78. Informe clínico del paciente: RZRM020101906017 de 8 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Madrid en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Dra. Laguna +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente 041010000011 significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Reinaldo Manjón Malo Barcelona Servicio de Endocrinología y Nutrición Valencia km 12,500 28905 Bilbao - Madrid (Barcelona) Correo electrónico: quintanasalome@example.net {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|281.3 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English image_classifier_vit_rust_image_classification_2 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_2 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_2` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_2_en_4.1.0_3.0_1660166968473.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_2_en_4.1.0_3.0_1660166968473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_2", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_2", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_2| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Employee benefit plans Clause Binary Classifier (md) author: John Snow Labs name: legclf_employee_benefit_plans_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `employee-benefit-plans` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `employee-benefit-plans` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_employee_benefit_plans_md_en_1.0.0_3.0_1669376476318.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_employee_benefit_plans_md_en_1.0.0_3.0_1669376476318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_employee_benefit_plans_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[employee-benefit-plans]| |[other]| |[other]| |[employee-benefit-plans]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_employee_benefit_plans_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support employee-benefit-plans 0.97 1.00 0.98 28 other 1.00 0.97 0.99 39 accuracy 0.99 67 macro avg 0.98 0.99 0.98 67 weighted avg 0.99 0.99 0.99 67 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_base_finetuned_scrambled_squad_5 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-5` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_5_en_4.3.0_3.0_1674216944002.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_scrambled_squad_5_en_4.3.0_3.0_1674216944002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_5","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_scrambled_squad_5","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_scrambled_squad_5| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-5 --- layout: model title: NER Model Finder with Sentence Entity Resolvers (sbert_jsl_medium_uncased) author: John Snow Labs name: sbertresolve_ner_model_finder date: 2021-11-24 tags: [ner, licensed, en, clinical, entity_resolver] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.2 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities (NER labels) to the most appropriate NER model using `sbert_jsl_medium_uncased` Sentence Bert Embeddings. Given the entity name, it will return a list of pretrained NER models having that entity or similar ones. ## Predicted Entities `NER Model Names` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_ner_model_finder_en_3.3.2_2.4_1637764208798.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_ner_model_finder_en_3.3.2_2.4_1637764208798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_uncased","en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") ner_model_finder = SentenceEntityResolverModel\ .pretrained("sbertresolve_ner_model_finder", "en", "clinical/models")\ .setInputCols(["ner_chunk", "sbert_embeddings"])\ .setOutputCol("model_names")\ .setDistanceFunction("EUCLIDEAN") ner_model_finder_pipelineModel = PipelineModel(stages = [documentAssembler, sbert_embedder, ner_model_finder]) light_pipeline = LightPipeline(ner_model_finder_pipelineModel) annotations = light_pipeline.fullAnnotate("medication") ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val ner_model_finder = SentenceEntityResolverModel .pretrained("sbertresolve_ner_model_finder", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("model_names") .setDistanceFunction("EUCLIDEAN") val ner_model_finder_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, ner_model_finder)) val light_pipeline = LightPipeline(ner_model_finder_pipelineModel) val annotations = light_pipeline.fullAnnotate("medication") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.ner.model_finder").predict("""Put your text here.""") ```

## Results ```bash +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |entity |models |all_models |resolutions | +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |medication|['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_ade_clinical', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']|['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_ade_clinical', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']:::['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_clinical_large', 'ner_healthcare', 'ner_jsl_enriched', 'ner_clinical', 'ner_jsl_slim', 'ner_covid_trials', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_events_admission_clinical', 'ner_events_healthcare', 'ner_events_clinical', 'ner_jsl_greedy']:::['ner_medmentions_coarse']:::['ner_jsl_enriched', 'ner_covid_trials', 'ner_jsl', 'ner_medmentions_coarse']:::['ner_drugs']:::['ner_clinical_icdem', 'ner_medmentions_coarse']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_medmentions_coarse', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_medmentions_coarse', 'ner_radiology_wip_clinical', 'ner_jsl_slim', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy', 'ner_radiology']:::['ner_medmentions_coarse','ner_clinical_icdem']:::['ner_posology_experimental']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_measurements_clinical', 'ner_radiology_wip_clinical', 'ner_radiology']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_posology_small', 'ner_jsl_enriched', 'ner_posology_experimental', 'ner_posology_large', 'ner_posology_healthcare', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_posology_greedy', 'ner_posology', 'ner_jsl_greedy']:::['ner_covid_trials', 'ner_medmentions_coarse', 'jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_deid_subentity_augmented', 'ner_deid_subentity_glove', 'ner_deidentify_dl', 'ner_deid_enriched']:::['jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_covid_trials', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_medmentions_coarse', 'jsl_rd_ner_wip_greedy_clinical', 'ner_jsl_enriched', 'ner_jsl', 'jsl_ner_wip_modifier_clinical', 'ner_jsl_greedy']:::['ner_chemd_clinical']|medication:::drug:::treatment:::therapeutic procedure:::drug ingredient:::drug chemical:::diagnostic aid:::substance:::medical device:::diagnostic procedure:::administration:::measurement:::drug strength:::physiological reaction:::patient:::vaccine:::psychological condition:::abbreviation| +----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_ner_model_finder| |Compatibility:|Healthcare NLP 3.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sbert_embeddings]| |Output Labels:|[models]| |Language:|en| |Case sensitive:|false| --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_dl6 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-dl6` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_dl6_en_4.3.0_3.0_1675123262536.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_dl6_en_4.3.0_3.0_1675123262536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_dl6","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_dl6","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_dl6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|62.3 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-dl6 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Financial NER (Headers / Subheaders) author: John Snow Labs name: finner_headers date: 2022-08-29 tags: [en, finance, ner, headers, splitting, sections, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Named Entity Recognition model, which will help you split long financial documents into smaller sections. To do that, it detects Headers and Subheaders of different sections. You can then use the beginning and end information in the metadata to retrieve the text between those headers. This model has been trained on 10-K filings, with the following HEADER and SUBHEADERS annotation guidelines: - PART I, PART II, etc are HEADERS - Item 1, Item 2, etc are also HEADERS - Item 1A, 2B, etc are SUBHEADERS - 1., 2., 2.1, etc. are SUBHEADERS - Other kind of short section names are also SUBHEADERS For more information about long document splitting, see [this](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb) workshop entry. ## Predicted Entities `HEADER`, `SUBHEADER` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_HEADERS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_headers_en_1.0.0_3.2_1661771922923.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_headers_en_1.0.0_3.2_1661771922923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+-----------+----------+ | token| ner_label|confidence| +-----------+-----------+----------+ | 2| O| 0.576| | .| O| 0.9612| |Definitions|B-SUBHEADER| 0.9993| | .| O| 0.9755| | For| O| 0.9966| | purposes| O| 0.9863| | of| O| 0.9878| | this| O| 0.9974| | Agreement| O| 0.9994| | ,| O| 0.9999| | the| O| 1.0| | following| O| 1.0| | terms| O| 1.0| | have| O| 1.0| | the| O| 1.0| | meanings| O| 1.0| | ascribed| O| 1.0| | thereto| O| 1.0| | in| O| 1.0| | this| O| 1.0| | Section| O| 0.9985| | 1| O| 0.9999| | .| O| 0.9972| | 2| O| 0.9686| | .| O| 0.9834| |Appointment|B-SUBHEADER| 0.767| | as|I-SUBHEADER| 0.9479| | Reseller|I-SUBHEADER| 0.8429| | .| O| 0.9944| | 2.1|B-SUBHEADER| 0.6278| |Appointment|I-SUBHEADER| 0.6599| | .| O| 0.9972| | The| O| 0.9987| | Company| O| 0.9889| | hereby| O| 0.9914| | [***]| O| 0.9996| | .| O| 0.9999| | Allscripts| O| 0.9843| | may| O| 0.9989| | also| O| 0.9967| | disclose| O| 0.9949| | Company's| O| 0.9976| | pricing| O| 0.9999| |information| O| 0.9999| | relating| O| 0.9999| | to| O| 0.9998| | its| O| 0.9992| | Merchant| O| 0.9671| | Processing| O| 0.8411| | Services| O| 0.9662| +-----------+-----------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_headers| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on 10-k filings ## Benchmarking ```bash label tp fp fn prec rec f1 I-HEADER 2835 9 8 0.996835 0.9971860 0.9970107 B-SUBHEADER 963 135 131 0.877049 0.8802559 0.87864965 I-SUBHEADER 2573 219 152 0.921561 0.9442202 0.9327533 B-HEADER 425 1 1 0.997652 0.9976526 0.9976526 Macro-average 6796 364 292 0.948274 0.9548287 0.95154047 Micro-average 6796 364 292 0.949162 0.9588036 0.9539584 ``` --- layout: model title: Estonian XlmRoBertaForQuestionAnswering (from anukaver) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_est_qa date: 2022-06-23 tags: [open_source, question_answering, xlmroberta] task: Question Answering language: et edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-est-qa` is a Estonian model originally trained by `anukaver`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_est_qa_et_4.0.0_3.0_1655992235241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_est_qa_et_4.0.0_3.0_1655992235241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_est_qa","et") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_est_qa","et") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("et.answer_question.xlm_roberta.by_anukaver").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_est_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|et| |Size:|883.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anukaver/xlm-roberta-est-qa --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squadv1-x2.44-f87.7-d26-hybrid-filled-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1_en_4.0.0_3.0_1654181609802.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1_en_4.0.0_3.0_1654181609802.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_x2.32_f86.6_d15_hybrid_v1.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squadv1_x2.44_f87.7_d26_hybrid_filled_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|174.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squadv1-x2.44-f87.7-d26-hybrid-filled-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: Fast Neural Machine Translation Model from English to Japanese author: John Snow Labs name: opus_mt_en_jap date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, jap, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `jap` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_jap_xx_2.7.0_2.4_1609168310758.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_jap_xx_2.7.0_2.4_1609168310758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_jap| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RoBERTa Embeddings (from jackaduma) author: John Snow Labs name: roberta_embeddings_SecRoBERTa date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `SecRoBERTa` is a English model orginally trained by `jackaduma`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_SecRoBERTa_en_3.4.2_3.0_1649946774316.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_SecRoBERTa_en_3.4.2_3.0_1649946774316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_SecRoBERTa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|314.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/jackaduma/SecRoBERTa - https://github.com/jackaduma/SecBERT/ - https://github.com/kbandla/APTnotes - https://stucco.github.io/data/ - https://ebiquity.umbc.edu/_file_directory_/papers/943.pdf - https://competitions.codalab.org/competitions/17262 - https://github.com/allenai/scibert - https://github.com/jackaduma/SecBERT - https://github.com/jackaduma/SecBERT --- layout: model title: Social Determinants of Health (clinical_large) author: John Snow Labs name: ner_sdoh_emb_clinical_large_wip date: 2023-04-17 tags: [en, clinical_large, social_determinants, public_health, ner, sdoh, pyspark_30, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts terminology related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Access_To_Care`, `Age`, `Alcohol`, `Chidhood_Event`, `Communicable_Disease`, `Community_Safety`, `Diet`, `Disability`, `Eating_Disorder`, `Education`, `Employment`, `Environmental_Condition`, `Exercise`, `Family_Member`, `Financial_Status`, `Food_Insecurity`, `Gender`, `Geographic_Entity`, `Healthcare_Institution`, `Housing`, `Hyperlipidemia`, `Hypertension`, `Income`, `Insurance_Status`, `Language`, `Legal_Issues`, `Marital_Status`, `Mental_Health`, `Obesity`, `Other_Disease`, `Other_SDoH_Keywords`, `Population_Group`, `Quality_Of_Life`, `Race_Ethnicity`, `Sexual_Activity`, `Sexual_Orientation`, `Smoking`, `Social_Exclusion`, `Social_Support`, `Spiritual_Beliefs`, `Substance_Duration`, `Substance_Frequency`, `Substance_Quantity`, `Substance_Use`, `Transportation`, `Violence_Or_Abuse` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_NER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_emb_clinical_large_wip_en_4.3.2_3.0_1681756284245.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_emb_clinical_large_wip_en_4.3.2_3.0_1681756284245.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_emb_clinical_large_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = [["Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week."]] data = spark.createDataFrame(sample_texts).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_emb_clinical_large_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_emb_clinical_large_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = [["Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week."]] data = spark.createDataFrame(sample_texts).toDF("text") result = pipeline.fit(data).transform(data) ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_emb_clinical_large_wip| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.0 MB| |Dependencies:|embeddings_clinical_large| ## References Internal SHOP Project ## Benchmarking ```bash label precision recall f1-score support Employment 0.94 0.96 0.95 2075 Social_Support 0.91 0.90 0.90 658 Other_SDoH_Keywords 0.82 0.87 0.85 259 Healthcare_Institution 0.99 0.95 0.97 781 Alcohol 0.96 0.97 0.96 258 Gender 0.99 0.99 0.99 4957 Other_Disease 0.89 0.94 0.91 583 Access_To_Care 0.86 0.88 0.87 520 Mental_Health 0.89 0.81 0.85 494 Age 0.92 0.96 0.94 433 Marital_Status 1.00 1.00 1.00 92 Substance_Quantity 0.88 0.86 0.87 58 Substance_Use 0.91 0.97 0.94 192 Family_Member 0.97 0.99 0.98 2094 Financial_Status 0.86 0.65 0.74 124 Race_Ethnicity 0.93 0.93 0.93 27 Insurance_Status 0.93 0.87 0.90 85 Spiritual_Beliefs 0.86 0.81 0.83 52 Housing 0.88 0.85 0.87 400 Geographic_Entity 0.86 0.88 0.87 113 Disability 0.93 0.93 0.93 44 Quality_Of_Life 0.89 0.75 0.81 67 Income 0.89 0.77 0.83 31 Education 0.85 0.88 0.86 58 Transportation 0.86 0.89 0.88 57 Legal_Issues 0.72 0.91 0.80 47 Smoking 0.98 0.97 0.98 66 Substance_Frequency 0.93 0.75 0.83 57 Hypertension 1.00 1.00 1.00 21 Violence_Or_Abuse 0.83 0.62 0.71 63 Exercise 0.96 0.88 0.92 57 Diet 0.95 0.87 0.91 70 Sexual_Orientation 0.68 1.00 0.81 13 Language 0.89 0.73 0.80 22 Social_Exclusion 0.96 0.90 0.93 29 Substance_Duration 0.75 0.85 0.80 39 Communicable_Disease 1.00 0.84 0.91 31 Chidhood_Event 0.88 0.61 0.72 23 Community_Safety 0.95 0.93 0.94 44 Population_Group 0.89 0.62 0.73 13 Hyperlipidemia 0.78 1.00 0.88 7 Food_Insecurity 1.00 0.93 0.96 29 Eating_Disorder 0.67 0.92 0.77 13 Sexual_Activity 0.84 0.90 0.87 29 Environmental_Condition 1.00 1.00 1.00 20 Obesity 1.00 1.00 1.00 12 micro-avg 0.95 0.95 0.95 15217 macro-avg 0.90 0.88 0.88 15217 weighted-avg 0.95 0.95 0.95 15217 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Nadav) author: John Snow Labs name: roberta_qa_base_squad_finetuned_on_runaways date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad-finetuned-on-runaways-en` is a English model originally trained by `Nadav`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad_finetuned_on_runaways_en_4.3.0_3.0_1674218728000.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad_finetuned_on_runaways_en_4.3.0_3.0_1674218728000.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad_finetuned_on_runaways","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad_finetuned_on_runaways","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_squad_finetuned_on_runaways| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Nadav/roberta-base-squad-finetuned-on-runaways-en --- layout: model title: Chinese T5ForConditionalGeneration Cased model (from IDEA-CCNL) author: John Snow Labs name: t5_randeng_77m_multitask_chinese date: 2023-01-30 tags: [zh, open_source, t5, tensorflow] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Randeng-T5-77M-MultiTask-Chinese` is a Chinese model originally trained by `IDEA-CCNL`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_randeng_77m_multitask_chinese_zh_4.3.0_3.0_1675098367899.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_randeng_77m_multitask_chinese_zh_4.3.0_3.0_1675098367899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_randeng_77m_multitask_chinese","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_randeng_77m_multitask_chinese","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_randeng_77m_multitask_chinese| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|349.2 MB| ## References - https://huggingface.co/IDEA-CCNL/Randeng-T5-77M-MultiTask-Chinese - https://github.com/IDEA-CCNL/Fengshenbang-LM - https://fengshenbang-doc.readthedocs.io/ - http://jmlr.org/papers/v21/20-074.html - https://github.com/IDEA-CCNL/Fengshenbang-LM/ - https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/pretrain_t5 - https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/mt5_summary - https://github.com/IDEA-CCNL/Fengshenbang-LM/ - https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/pretrain_t5 - https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/mt5_summary - https://arxiv.org/abs/2209.02970 - https://arxiv.org/abs/2209.02970 - https://github.com/IDEA-CCNL/Fengshenbang-LM/ - https://github.com/IDEA-CCNL/Fengshenbang-LM/ --- layout: model title: English BertForQuestionAnswering Cased model (from clementgyj) author: John Snow Labs name: bert_qa_finetuned_squad_50k date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-50k` is a English model originally trained by `clementgyj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_finetuned_squad_50k_en_4.0.0_3.0_1657186899398.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_finetuned_squad_50k_en_4.0.0_3.0_1657186899398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_finetuned_squad_50k","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_finetuned_squad_50k","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_finetuned_squad_50k| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/clementgyj/bert-finetuned-squad-50k --- layout: model title: French CamemBert Embeddings (from fjluque) author: John Snow Labs name: camembert_embeddings_fjluque_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `fjluque`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_fjluque_generic_model_fr_3.4.4_3.0_1653988580244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_fjluque_generic_model_fr_3.4.4_3.0_1653988580244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_fjluque_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/fjluque/dummy-model --- layout: model title: English asr_Urdu_repo TFWav2Vec2ForCTC from bilalahmed15 author: John Snow Labs name: asr_Urdu_repo date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Urdu_repo` is a English model originally trained by bilalahmed15. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Urdu_repo_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Urdu_repo_en_4.2.0_3.0_1664107184096.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Urdu_repo_en_4.2.0_3.0_1664107184096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Urdu_repo", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Urdu_repo", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Urdu_repo| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Finnish BERT Sentence Embeddings (Base Cased) author: John Snow Labs name: sent_bert_finnish_cased date: 2020-08-31 task: Embeddings language: fi edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, fi] supported: true deprecated: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A version of Google's BERT deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. `FinBERT` features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words. `FinBERT` has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train `FinBERT`. These features allow `FinBERT` to outperform not only Multilingual BERT but also all previously proposed models when fine-tuned for Finnish natural language processing tasks. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_finnish_cased_fi_2.6.0_2.4_1598897560014.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_finnish_cased_fi_2.6.0_2.4_1598897560014.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_bert_finnish_cased", "fi") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Vihaan syöpää', 'antibiootit eivät ole kipulääkkeitä']], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_bert_finnish_cased", "fi") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("Vihaan syöpää","antibiootit eivät ole kipulääkkeitä").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Vihaan syöpää","antibiootit eivät ole kipulääkkeitä"] embeddings_df = nlu.load('fi.embed_sentence.bert.cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence fi_embed_sentence_bert_cased_embeddings Vihaan syöpää [-0.32807931303977966, -0.18222537636756897, 0... antibiootit eivät ole kipulääkkeitä [-0.192955881357193, -0.11151257902383804, 0.7... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_finnish_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[fi]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from https://github.com/TurkuNLP/FinBERT --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from SEISHIN) author: John Snow Labs name: distilbert_qa_seishin_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `SEISHIN`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_seishin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769121878.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_seishin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769121878.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_seishin_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_seishin_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_seishin_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SEISHIN/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate English to Bislama Pipeline author: John Snow Labs name: translate_en_bi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, bi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `bi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_bi_xx_2.7.0_2.4_1609698734459.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_bi_xx_2.7.0_2.4_1609698734459.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_bi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_bi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.bi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_bi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Extract Demographic Entities from Social Determinants of Health Texts author: John Snow Labs name: ner_sdoh_demographics_wip date: 2023-02-10 tags: [licensed, clinical, social_determinants, en, ner, demographics, sdoh, public_health] task: Named Entity Recognition language: en nav_key: modelsr edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts demographic information related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Family_Member`, `Age`, `Gender`, `Geographic_Entity`, `Race_Ethnicity`, `Language`, `Spiritual_Beliefs` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_NER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_demographics_wip_en_4.2.8_3.0_1675998706136.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_demographics_wip_en_4.2.8_3.0_1675998706136.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_demographics_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = ["SOCIAL HISTORY: He is a former tailor from Korea.", "He lives alone,single and no children.", "Pt is a 61 years old married, Caucasian, Catholic woman. Pt speaks English reasonably well."] data = spark.createDataFrame(sample_texts, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_demographics_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("Pt is a 61 years old married, Caucasian, Catholic woman. Pt speaks English reasonably well.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-----------------+-----+---+------------+ |ner_label |begin|end|chunk | +-----------------+-----+---+------------+ |Gender |16 |17 |He | |Geographic_Entity|43 |47 |Korea | |Gender |0 |1 |He | |Family_Member |29 |36 |children | |Age |8 |19 |61 years old| |Race_Ethnicity |30 |38 |Caucasian | |Spiritual_Beliefs|41 |48 |Catholic | |Gender |50 |54 |woman | |Language |67 |73 |English | +-----------------+-----+---+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_demographics_wip| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|858.4 KB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Age 1346.0 73.0 74.0 1420.0 0.948555 0.947887 0.948221 Spiritual_Beliefs 100.0 13.0 16.0 116.0 0.884956 0.862069 0.873362 Family_Member 4468.0 134.0 43.0 4511.0 0.970882 0.990468 0.980577 Race_Ethnicity 56.0 0.0 13.0 69.0 1.000000 0.811594 0.896000 Gender 9825.0 67.0 247.0 10072.0 0.993227 0.975477 0.984272 Geographic_Entity 225.0 9.0 29.0 254.0 0.961538 0.885827 0.922131 Language 51.0 9.0 5.0 56.0 0.850000 0.910714 0.879310 ``` --- layout: model title: Explain Document ML Pipeline for English author: John Snow Labs name: explain_document_ml date: 2022-06-24 tags: [open_source, english, explain_document_ml, pipeline, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_ml is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_ml_en_4.0.0_3.0_1656066222624.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_ml_en_4.0.0_3.0_1656066222624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | document | sentence | token | spell | lemmas | stems | pos | |---:|:---------------------------------|:---------------------------------|:-------------------------------------------------|:------------------------------------------------|:------------------------------------------------|:-----------------------------------------------|:---------------------------------------| | 0 | ['Hello fronm John Snwow Labs!'] | ['Hello fronm John Snwow Labs!'] | ['Hello', 'fronm', 'John', 'Snwow', 'Labs', '!'] | ['Hello', 'front', 'John', 'Snow', 'Labs', '!'] | ['Hello', 'front', 'John', 'Snow', 'Labs', '!'] | ['hello', 'front', 'john', 'snow', 'lab', '!'] | ['UH', 'NN', 'NNP', 'NNP', 'NNP', '.'] || | document | sentence | token | spell | lemmas | stems | pos | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_ml| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|9.6 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - NorvigSweetingModel - LemmatizerModel - Stemmer - PerceptronModel --- layout: model title: Legal Tariff Policy Document Classifier (EURLEX) author: John Snow Labs name: legclf_tariff_policy_bert date: 2023-03-06 tags: [en, legal, classification, clauses, tariff_policy, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_tariff_policy_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Tariff_Policy or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Tariff_Policy`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_tariff_policy_bert_en_1.0.0_3.0_1678111753015.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_tariff_policy_bert_en_1.0.0_3.0_1678111753015.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_tariff_policy_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Tariff_Policy]| |[Other]| |[Other]| |[Tariff_Policy]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_tariff_policy_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.82 0.85 0.83 969 Tariff_Policy 0.87 0.85 0.86 1175 accuracy - - 0.85 2144 macro-avg 0.85 0.85 0.85 2144 weighted-avg 0.85 0.85 0.85 2144 ``` --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-32-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1657185004862.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1657185004862.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-32-finetuned-squad-seed-4 --- layout: model title: Detect Subentity PHI for Deidentification (Arabic) author: John Snow Labs name: ner_deid_subentity date: 2023-05-31 tags: [licensed, clinical, ner, deidentification, arabic, ar] task: Named Entity Recognition language: ar edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Arabic) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities. This NER model is trained with a combination of custom datasets, and several data augmentation mechanisms. This model Word2Vec Arabic Clinical Embeddings. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `STREET`, `USERNAME`, `SEX`, `IDNUM`, `EMAIL`, `ZIP`, `MEDICALRECORD`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ar_4.4.2_3.0_1685559675615.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ar_4.4.2_3.0_1685559675615.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------------------------+----------------+ |chunk |ner_label| +------------------------------------------------+---------------+ |محمد |DOCTOR | |55 سنة |AGE | |15/05/2000 |DATE | |الرباط |CITY | |0610948235 |PHONE | |mohamedmell@gmail.com |EMAIL | +------------------------------------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ar| |Size:|15.0 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 196.0 26.0 32.0 228.0 0.8829 0.8596 0.8711 HOSPITAL 193.0 41.0 37.0 230.0 0.8248 0.8391 0.8319 DATE 877.0 14.0 8.0 885.0 0.9843 0.991 0.9876 ORGANIZATION 41.0 11.0 6.0 47.0 0.7885 0.8723 0.8283 CITY 260.0 8.0 5.0 265.0 0.9701 0.9811 0.9756 STREET 103.0 3.0 0.0 103.0 0.9717 1.0 0.9856 USERNAME 8.0 0.0 0.0 8.0 1.0 1.0 1.0 SEX 300.0 9.0 69.0 369.0 0.9709 0.813 0.885 IDNUM 13.0 1.0 0.0 13.0 0.9286 1.0 0.963 EMAIL 112.0 5.0 0.0 112.0 0.9573 1.0 0.9782 ZIP 80.0 4.0 0.0 80.0 0.9524 1.0 0.9756 MEDICALRECORD 17.0 1.0 0.0 17.0 0.9444 1.0 0.9714 PROFESSION 303.0 27.0 32.0 335.0 0.9182 0.9045 0.9113 PHONE 38.0 4.0 2.0 40.0 0.9048 0.95 0.9268 COUNTRY 158.0 10.0 8.0 166.0 0.9405 0.9518 0.9461 DOCTOR 440.0 23.0 34.0 474.0 0.9503 0.9283 0.9392 AGE 610.0 18.0 7.0 617.0 0.9713 0.9887 0.9799 macro - - - - - - 0.9386 micro - - - - - - 0.9434 ``` --- layout: model title: Word2Vec Embeddings in Minangkabau (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, min, open_source] task: Embeddings language: min edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_min_3.4.1_3.0_1647446021762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_min_3.4.1_3.0_1647446021762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_squad2_finetuned_selqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_squad2_finetuned_selqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_Teepika").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_squad2_finetuned_selqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Teepika/roberta-base-squad2-finetuned-selqa --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265899` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899_en_4.0.0_3.0_1655984561078.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899_en_4.0.0_3.0_1655984561078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265899").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265899| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265899 --- layout: model title: Stopwords Remover for Lithuanian language (1314 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, lt, open_source] task: Stop Words Removal language: lt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_lt_3.4.1_3.0_1646673057631.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_lt_3.4.1_3.0_1646673057631.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------+ |result | +------------------+ |[nesate, geresnis]| +------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|lt| |Size:|4.8 KB| --- layout: model title: English RobertaForQuestionAnswering (from billfrench) author: John Snow Labs name: roberta_qa_cyberlandr_door date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cyberlandr-door` is a English model originally trained by `billfrench`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cyberlandr_door_en_4.0.0_3.0_1655728103928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cyberlandr_door_en_4.0.0_3.0_1655728103928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_cyberlandr_door","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_cyberlandr_door","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.by_billfrench").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cyberlandr_door| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|413.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/billfrench/cyberlandr-door --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from arunkumar629) author: John Snow Labs name: distilbert_qa_arunkumar629_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `arunkumar629`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_arunkumar629_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769951769.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_arunkumar629_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769951769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_arunkumar629_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_arunkumar629_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_arunkumar629_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/arunkumar629/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you TFWav2Vec2ForCTC from project2you author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you` is a English model originally trained by project2you. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_en_4.2.0_3.0_1664110153624.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_en_4.2.0_3.0_1664110153624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from Intel) author: John Snow Labs name: bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-squadv1.1-sparse-80-1x4-block-pruneofa` is a English model orginally trained by `Intel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa_en_4.0.0_3.0_1654536766214.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa_en_4.0.0_3.0_1654536766214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.large_uncased_sparse_80_1x4_block_pruneofa.by_Intel").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_squadv1.1_sparse_80_1x4_block_pruneofa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|437.9 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Intel/bert-large-uncased-squadv1.1-sparse-80-1x4-block-pruneofa - https://arxiv.org/abs/2111.05754 - https://github.com/IntelLabs/Model-Compression-Research-Package/tree/main/research/prune-once-for-all --- layout: model title: Detect Founding / Listing dates in texts (small) author: John Snow Labs name: finner_wiki_founding_dates date: 2023-01-15 tags: [listing, founding, establishment, dates, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model, aimed to detect Establishment (Founding) and Listing dates of Companies. It was trained with wikipedia texts about companies. ## Predicted Entities `FOUNDING_DATE`, `LISTING_DATE`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_wiki_founding_dates_en_1.0.0_3.0_1673798045941.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_wiki_founding_dates_en_1.0.0_3.0_1673798045941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python text = "The Toro Company, formerly known as the Toro Motor Company, is an American company founded in 1980. It was listed on the NASDAQ Global Market in August 2000. It design and operates lawn mowers and snow blowers and irrigation system supplies." df = spark.createDataFrame([[text]]).toDF("text") documenter = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) chunks = finance.NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") ner = finance.NerModel().pretrained("finner_wiki_founding_dates", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") pipe = nlp.Pipeline(stages=[documenter, sentencizer, tokenizer, embeddings, ner, chunks]) model = pipe.fit(df) res = model.transform(df) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.begin, res.ner_chunk.end, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['3']['sentence']").alias("sentence_id"), F.expr("cols['0']").alias("chunk"), F.expr("cols['2']").alias("end"), F.expr("cols['3']['entity']").alias("ner_label"))\ .filter("ner_label!='O'")\ .show(truncate=False) ```

## Results ```bash +-----------+-----------+---+-------------+ |sentence_id|chunk |end|ner_label | +-----------+-----------+---+-------------+ |0 |1980 |97 |FOUNDING_DATE| |1 |August 2000|155|LISTING_DATE | +-----------+-----------+---+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_wiki_founding_dates| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.1 MB| ## References Wikipedia ## Benchmarking ```bash label tp fp fn prec rec f1 B-LISTING_DATE 10 0 4 1.0 0.71428573 0.8333334 B-FOUNDING_DATE 18 3 2 0.85714287 0.9 0.87804884 I-LISTING_DATE 8 0 1 1.0 0.8888889 0.94117653 Macro-average 36 4 9 4 0.9 0.8 0.8470588 Micro-average 36 4 9 4 0.9 0.8 0.8470588 ``` --- layout: model title: Document Visual Question Answering with DONUT author: John Snow Labs name: docvqa_donut_base date: 2023-01-17 tags: [en, licensed] task: Document Visual Question Answering language: en nav_key: models edition: Visual NLP 4.3.0 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Document understanding transformer (Donut) model pretrained for Document Visual Question Answering (DocVQA) task in the dataset is from Document Visual Question Answering [competition](https://rrc.cvc.uab.es/?ch=17) and consists of 50K questions defined on more than 12K documents. Donut is a new method of document understanding that utilizes an OCR-free end-to-end Transformer model. Donut does not require off-the-shelf OCR engines/APIs, yet it shows state-of-the-art performances on various visual document understanding tasks, such as visual document classification or information extraction (a.k.a. document parsing). Paper link [OCR-free Document Understanding Transformer](https://arxiv.org/abs/2111.15664) developed by Geewook Kim, Teakgyu Hong, Moonbin Yim, Jeongyeon Nam, Jinyoung Park, Jinyeong Yim, Wonseok Hwang, Sangdoo Yun, Dongyoon Han and Seunghyun Park. DocVQA seeks to inspire a “purpose-driven” point of view in Document Analysis and Recognition research, where the document content is extracted and used to respond to high-level tasks defined by the human consumers of this information. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/VISUAL_QUESTION_ANSWERING/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/jupyter/Cards/SparkOcrVisualQuestionAnswering.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/docvqa_donut_base_en_4.3.0_3.0_1673269990044.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/docvqa_donut_base_en_4.3.0_3.0_1673269990044.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage()\ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) visual_question_answering = VisualQuestionAnswering()\ .pretrained("docvqa_donut_base", "en", "clinical/ocr")\ .setInputCol(["image"])\ .setOutputCol("answers")\ .setQuestionsCol("questions") # OCR pipeline pipeline = PipelineModel(stages=[ binary_to_image, visual_question_answering ]) test_image_path = pkg_resources.resource_filename('sparkocr', 'resources/ocr/vqa/agenda.png') bin_df = spark.read.format("binaryFile").load(test_image_path) questions = [["When it finish the Coffee Break?", "Who is giving the Introductory Remarks?", "Who is going to take part of the individual interviews?"]] questions_df = spark.createDataFrame([questions]) questions_df = questions_df.withColumnRenamed("_1", "questions") image_and_questions = bin_df.join(questions_df) results = pipeline.transform(image_and_questions).cache() results.select(results.answers).show(truncate=False) ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val visual_question_answering = VisualQuestionAnswering() .pretrained("docvqa_donut_base", "en", "clinical/ocr") .setInputCol(Array("image")) .setOutputCol("answers") .setQuestionsCol("questions") # OCR pipeline val pipeline = new PipelineModel().setStages(Array( binary_to_image, visual_question_answering)) val test_image_path = pkg_resources.resource_filename("sparkocr", "resources/ocr/vqa/agenda.png") val bin_df = spark.read.format("binaryFile").load(test_image_path) val questions = Array("When it finish the Coffee Break?", "Who is giving the Introductory Remarks?", "Who is going to take part of the individual interviews?") val questions_df = spark.createDataFrame(Array(questions)) val questions_df = questions_df.withColumnRenamed("_1", "questions") val image_and_questions = bin_df.join(questions_df) val results = pipeline.transform(image_and_questions).cache() results.select(results.answers).show(truncate=False) ```

## Example ### Input: ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |questions | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[ When it finish the Coffee Break?, Who is giving the Introductory Remarks?, Who is going to take part of the individual interviews? +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` ![Screenshot](/assets/images/examples_ocr/image12.png) ### Output: ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |answers | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[ When it finish the Coffee Break? -> 11:44 to 11:39 a.m., Who is giving the Introductory Remarks? -> lee a. waller, trrf vice presi- dent, Who is going to take part of the individual interviews? -> trrf treasurer]| +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` ## Model Information {:.table-model} |---|---| |Model Name:|docvqa_donut_base| |Type:|ocr| |Compatibility:|Visual NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: ELECTRA Sentence Embeddings(ELECTRA Base) author: John Snow Labs name: sent_electra_base_uncased date: 2020-08-27 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ELECTRA is a BERT-like model that is pre-trained as a discriminator in a set-up resembling a generative adversarial network (GAN). It was originally published by: Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning: [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/forum?id=r1xMH1BtvB), ICLR 2020. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_electra_base_uncased_en_2.6.0_2.4_1598489784655.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_electra_base_uncased_en_2.6.0_2.4_1598489784655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_electra_base_uncased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_electra_base_uncased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.electra_base_uncased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_electra_base_uncased_embeddings I hate cancer [0.18555310368537903, -0.1990899294614792, 0.2... Antibiotics aren't painkiller [-0.23764970898628235, -0.21351191401481628, -... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_electra_base_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/google/electra_base/2 --- layout: model title: English image_classifier_vit_base_xray_pneumonia ViTForImageClassification from nickmuchi author: John Snow Labs name: image_classifier_vit_base_xray_pneumonia date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_xray_pneumonia` is a English model originally trained by nickmuchi. ## Predicted Entities `NORMAL`, `PNEUMONIA` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_xray_pneumonia_en_4.1.0_3.0_1660170972982.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_xray_pneumonia_en_4.1.0_3.0_1660170972982.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_xray_pneumonia", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_xray_pneumonia", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_xray_pneumonia| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForQuestionAnswering model (from Plimpton) author: John Snow Labs name: distilbert_qa_Plimpton_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Plimpton`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Plimpton_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724353175.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Plimpton_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724353175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Plimpton_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Plimpton_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Plimpton").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Plimpton_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Plimpton/distilbert-base-uncased-finetuned-squad --- layout: model title: XLM-RoBERTa 40-Language NER Pipeline author: John Snow Labs name: xlm_roberta_token_classifier_ner_40_lang_pipeline date: 2022-06-27 tags: [open_source, ner, token_classifier, xlm_roberta, multilang, "40", xx] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_token_classifier_ner_40_lang](https://nlp.johnsnowlabs.com/2021/09/28/xlm_roberta_token_classifier_ner_40_lang_xx.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_token_classifier_ner_40_lang_pipeline_xx_4.0.0_3.0_1656370754079.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_token_classifier_ner_40_lang_pipeline_xx_4.0.0_3.0_1656370754079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlm_roberta_token_classifier_ner_40_lang_pipeline", lang = "xx") pipeline.annotate(["My name is John and I work at John Snow Labs.", "انا اسمي احمد واعمل في ارامكو"]) ``` ```scala val pipeline = new PretrainedPipeline("xlm_roberta_token_classifier_ner_40_lang_pipeline", lang = "xx") pipeline.annotate(Array("My name is John and I work at John Snow Labs.", "انا اسمي احمد واعمل في ارامكو")) ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PER | |John Snow Labs|ORG | |احمد |PER | |ارامكو |ORG | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_token_classifier_ner_40_lang_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|xx| |Size:|967.7 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - XlmRoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: Pipeline to Detect Clinical Entities (ner_jsl_enriched) author: John Snow Labs name: ner_jsl_enriched_pipeline date: 2023-03-14 tags: [ner, licensed, clinical, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_enriched](https://nlp.johnsnowlabs.com/2021/10/22/ner_jsl_enriched_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_pipeline_en_4.3.0_3.2_1678779376891.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_pipeline_en_4.3.0_3.2_1678779376891.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_jsl_enriched_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_jsl_enriched_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_enriched.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.9993 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.9993 | | 2 | male | 38 | 41 | Gender | 0.999 | | 3 | 2 days | 52 | 57 | Duration | 0.8576 | | 4 | congestion | 62 | 71 | Symptom | 0.9892 | | 5 | mom | 75 | 77 | Gender | 0.9877 | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | 0.2232 | | 7 | nares | 135 | 139 | External_body_part_or_region | 0.87 | | 8 | she | 147 | 149 | Gender | 0.9965 | | 9 | mild | 168 | 171 | Modifier | 0.6063 | | 10 | problems with his breathing while feeding | 173 | 213 | Symptom | 0.610967 | | 11 | perioral cyanosis | 237 | 253 | Symptom | 0.5396 | | 12 | retractions | 258 | 268 | Symptom | 0.9941 | | 13 | One day ago | 272 | 282 | RelativeDate | 0.870133 | | 14 | mom | 285 | 287 | Gender | 0.9974 | | 15 | tactile temperature | 304 | 322 | Symptom | 0.43565 | | 16 | Tylenol | 345 | 351 | Drug_BrandName | 0.9926 | | 17 | Baby | 354 | 357 | Age | 0.9976 | | 18 | decreased p.o. intake | 377 | 397 | Symptom | 0.5397 | | 19 | His | 400 | 402 | Gender | 0.9998 | | 20 | 20 minutes q.2h. to 5 to 10 minutes | 439 | 473 | Duration | 0.3732 | | 21 | his | 488 | 490 | Gender | 0.9461 | | 22 | respiratory congestion | 492 | 513 | Symptom | 0.5958 | | 23 | He | 516 | 517 | Gender | 0.9998 | | 24 | tired | 550 | 554 | Symptom | 0.9595 | | 25 | fussy | 569 | 573 | Symptom | 0.8263 | | 26 | over the past 2 days | 575 | 594 | RelativeDate | 0.49826 | | 27 | albuterol | 637 | 645 | Drug_Ingredient | 0.993 | | 28 | ER | 671 | 672 | Clinical_Dept | 0.998 | | 29 | His | 675 | 677 | Gender | 0.9998 | | 30 | urine output has also decreased | 679 | 709 | Symptom | 0.26296 | | 31 | he | 721 | 722 | Gender | 0.9924 | | 32 | per 24 hours | 760 | 771 | Frequency | 0.4958 | | 33 | he | 778 | 779 | Gender | 0.9951 | | 34 | per 24 hours | 807 | 818 | Frequency | 0.484933 | | 35 | Mom | 821 | 823 | Gender | 0.999 | | 36 | diarrhea | 836 | 843 | Symptom | 0.9995 | | 37 | His | 846 | 848 | Gender | 0.9998 | | 38 | bowel | 850 | 854 | Internal_organ_or_component | 0.9675 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_enriched_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Swedish BertForQuestionAnswering model (from KB) author: John Snow Labs name: bert_qa_bert_base_swedish_cased_squad_experimental date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: sv edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-swedish-cased-squad-experimental` is a Swedish model orginally trained by `KB`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_swedish_cased_squad_experimental_sv_4.0.0_3.0_1654180645183.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_swedish_cased_squad_experimental_sv_4.0.0_3.0_1654180645183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_swedish_cased_squad_experimental","sv") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_swedish_cased_squad_experimental","sv") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("sv.answer_question.squad.bert.base_cased.by_KB").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_swedish_cased_squad_experimental| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|sv| |Size:|465.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KB/bert-base-swedish-cased-squad-experimental --- layout: model title: Language Detection & Identification Pipeline - 220 Languages author: John Snow Labs name: detect_language_220 date: 2020-12-05 task: [Pipeline Public, Language Detection, Sentence Detection] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, pipeline, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNN architectures in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This pipeline can detect the following languages: ## Predicted Entities `Achinese`, `Afrikaans`, `Tosk Albanian`, `Amharic`, `Aragonese`, `Old English`, `Arabic`, `Egyptian Arabic`, `Assamese`, `Asturian`, `Avaric`, `Aymara`, `Azerbaijani`, `South Azerbaijani`, `Bashkir`, `Bavarian`, `bat-smg`, `Central Bikol`, `Belarusian`, `Bulgarian`, `bh`, `Bengali`, `Tibetan`, `Bishnupriya`, `Breton`, `Russia Buriat`, `Catalan`, `Min Dong Chinese`, `Chechen`, `Cebuano`, `Central Kurdish (Soranî)`, `Corsican`, `Crimean Tatar`, `Czech`, `Kashubian`, `Chuvash`, `Welsh`, `Danish`, `German`, `Dimli (individual language)`, `Lower Sorbian`, `Dhivehi`, `Greek`, `eml`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Basque`, `Extremaduran`, `Persian`, `Finnish`, `fiu-vro`, `Faroese`, `French`, `Arpitan`, `Friulian`, `Frisian`, `Irish`, `Gagauz`, `Scottish Gaelic`, `Galician`, `Guarani`, `Konkani (Goan)`, `Gujarati`, `Manx`, `Hausa`, `Hakka Chinese`, `Hebrew`, `Hindi`, `Fiji Hindi`, `Upper Sorbian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Interlingue`, `Igbo`, `Ilocano`, `Ido`, `Icelandic`, `Italian`, `Japanese`, `Jamaican Patois`, `Lojban`, `Javanese`, `Georgian`, `Karakalpak`, `Kabyle`, `Kabardian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Komi-Permyak`, `Karachay-Balkar`, `Kölsch`, `Kurdish`, `Komi`, `Cornish`, `Kyrgyz`, `Latin`, `Ladino`, `Luxembourgish`, `Lezghian`, `Luganda`, `Limburgan`, `Ligurian`, `Lombard`, `Lingala`, `Lao`, `Northern Luri`, `Lithuanian`, `Latvian`, `Maithili`, `map-bms`, `Malagasy`, `Meadow Mari`, `Maori`, `Minangkabau`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Hill Mari`, `Maltese`, `Mirandese`, `Burmese`, `Erzya`, `Mazanderani`, `Nahuatl`, `Neapolitan`, `Low German (Low Saxon)`, `nds-nl`, `Nepali`, `Newari`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Narom`, `Pedi`, `Navajo`, `Occitan`, `Livvi`, `Oromo`, `Odia (Oriya)`, `Ossetian`, `Punjabi (Eastern)`, `Pangasinan`, `Kapampangan`, `Papiamento`, `Picard`, `Palatine German`, `Polish`, `Punjabi (Western)`, `Pashto`, `Portuguese`, `Quechua`, `Romansh`, `Romanian`, `roa-tara`, `Russian`, `Rusyn`, `Kinyarwanda`, `Sanskrit`, `Yakut`, `Sardinian`, `Sicilian`, `Scots`, `Sindhi`, `Northern Sami`, `Sinhala`, `Slovak`, `Slovenian`, `Shona`, `Somali`, `Albanian`, `Serbian`, `Saterland Frisian`, `Sundanese`, `Swedish`, `Swahili`, `Silesian`, `Tamil`, `Tulu`, `Telugu`, `Tetun`, `Tajik`, `Thai`, `Turkmen`, `Tagalog`, `Setswana`, `Tongan`, `Turkish`, `Tatar`, `Tuvinian`, `Udmurt`, `Uyghur`, `Ukrainian`, `Urdu`, `Uzbek`, `Venetian`, `Veps`, `Vietnamese`, `Vlaams`, `Volapük`, `Walloon`, `Waray`, `Wolof`, `Shanghainese`, `Xhosa`, `Mingrelian`, `Yiddish`, `Yoruba`, `Zeeuws`, `Chinese`, `zh-classical`, `zh-min-nan`, `zh-yue`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/LANGUAGE_DETECTOR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/detect_language_220_xx_2.7.0_2.4_1607185721383.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/detect_language_220_xx_2.7.0_2.4_1607185721383.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("detect_language_220", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("detect_language_220", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` {:.nlu-block} ```python import nlu text = ["French author who helped pioneer the science-fiction genre."] lang_df = nlu.load("xx.classify.lang.220").predict(text) lang_df ```

## Results ```bash {'document': ['French author who helped pioneer the science-fiction genre.'], 'language': ['en']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|detect_language_220| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - LanguageDetectorDL --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_name_all_904029577 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-name_all-904029577` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Name`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_all_904029577_en_4.3.1_3.0_1678783486467.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_all_904029577_en_4.3.1_3.0_1678783486467.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_all_904029577","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_all_904029577","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_name_all_904029577| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-name_all-904029577 --- layout: model title: English DistilBertForQuestionAnswering model (from aszidon) Custom author: John Snow Labs name: distilbert_qa_custom date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom_en_4.0.0_3.0_1654727944431.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom_en_4.0.0_3.0_1654727944431.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.custom.by_aszidon").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom --- layout: model title: Legal Time Clause Binary Classifier author: John Snow Labs name: legclf_time_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `time` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `time` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_time_clause_en_1.0.0_3.2_1660124089546.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_time_clause_en_1.0.0_3.2_1660124089546.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_time_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[time]| |[other]| |[other]| |[time]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_time_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.99 0.99 304 time 0.98 0.96 0.97 150 accuracy - - 0.98 454 macro-avg 0.98 0.98 0.98 454 weighted-avg 0.98 0.98 0.98 454 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from SauravMaheshkar) author: John Snow Labs name: distilbert_qa_base_cased_led_chaii date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_chaii_en_4.3.0_3.0_1672766429348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_chaii_en_4.3.0_3.0_1672766429348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_chaii","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_chaii| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/distilbert-base-cased-distilled-chaii --- layout: model title: Sentence Entity Resolver for Snomed Concepts, Body Structure Version author: John Snow Labs name: sbertresolve_snomed_bodyStructure_med date: 2021-07-08 tags: [snomed, en, entity_resolution, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical (anatomical structures) entities to Snomed codes (body structure version) using sentence embeddings. ## Predicted Entities Snomed Codes and their normalized definition with `sbert_jsl_medium_uncased ` embeddings. {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_snomed_bodyStructure_med_en_3.1.0_2.4_1625772026635.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_snomed_bodyStructure_med_en_3.1.0_2.4_1625772026635.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") jsl_sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbert_jsl_medium_uncased','en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel\ .pretrained("sbertresolve_snomed_bodyStructure_med", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("snomed_code") snomed_pipelineModel = PipelineModel( stages = [ documentAssembler, jsl_sbert_embedder, snomed_resolver]) snomed_lp = LightPipeline(snomed_pipelineModel) result = snomed_lp.fullAnnotate("Amputation stump") ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel .pretrained("sbertresolve_snomed_bodyStructure_med", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("snomed_code") val snomed_pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, snomed_resolver)) val snomed_lp = LightPipeline(snomed_pipelineModel) val result = snomed_lp.fullAnnotate("Amputation stump") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed_body_structure_med").predict("""Amputation stump""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:-----------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------| | 0 | amputation stump | 38033009 | [Amputation stump, Amputation stump of upper limb, Amputation stump of left upper limb, Amputation stump of lower limb, Amputation stump of left lower limb, Amputation stump of right upper limb, Amputation stump of right lower limb, ...]| ['38033009', '771359009', '771364008', '771358001', '771367001', '771365009', '771368006', ...] | ['0.0000', '0.0773', '0.0858', '0.0863', '0.0905', '0.0911', '0.0972', ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_snomed_bodyStructure_med| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[snomed_code]| |Language:|en| |Case sensitive:|true| ## Data Source https://www.snomed.org/ --- layout: model title: German asr_exp_w2v2t_vp_s962 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2t_vp_s962 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_vp_s962` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2t_vp_s962_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s962_de_4.2.0_3.0_1664111856089.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s962_de_4.2.0_3.0_1664111856089.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2t_vp_s962', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2t_vp_s962", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2t_vp_s962| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Anatomical References (biobert) author: John Snow Labs name: ner_anatomy_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect anatomical sites and references in medical text using pretrained NER model. ## Predicted Entities `tissue_structure`, `Organism_substance`, `Developing_anatomical_structure`, `Cell`, `Cellular_component`, `Immaterial_anatomical_entity`, `Organ`, `Pathological_formation`, `Organism_subdivision`, `Anatomical_system`, `Tissue` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ANATOMY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_biobert_en_3.0.0_3.0_1617260624773.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_biobert_en_3.0.0_3.0_1617260624773.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_anatomy_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_anatomy_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy.biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash +-------------------------------+-----+----+----+-----+---------+------+------+ | entity| tp| fp| fn|total|precision|recall| f1| +-------------------------------+-----+----+----+-----+---------+------+------+ | Organ| 53.0|17.0|12.0| 65.0| 0.7571|0.8154|0.7852| | Pathological_formation| 83.0|23.0|14.0| 97.0| 0.783|0.8557|0.8177| | Organism_substance| 42.0| 1.0|14.0| 56.0| 0.9767| 0.75|0.8485| | tissue_structure|131.0|28.0|49.0|180.0| 0.8239|0.7278|0.7729| | Cellular_component| 17.0| 0.0|20.0| 37.0| 1.0|0.4595|0.6296| | Tissue| 27.0| 4.0|16.0| 43.0| 0.871|0.6279|0.7297| | Anatomical_system| 15.0| 3.0| 8.0| 23.0| 0.8333|0.6522|0.7317| |Developing_anatomical_structure| 2.0| 1.0| 3.0| 5.0| 0.6667| 0.4| 0.5| | Immaterial_anatomical_entity| 7.0| 2.0| 6.0| 13.0| 0.7778|0.5385|0.6364| | Cell|180.0| 6.0|15.0|195.0| 0.9677|0.9231|0.9449| | Organism_subdivision| 11.0| 5.0|10.0| 21.0| 0.6875|0.5238|0.5946| +-------------------------------+-----+----+----+-----+---------+------+------+ +------------------+ | macro| +------------------+ |0.7264701979913192| +------------------+ +------------------+ | micro| +------------------+ |0.8108878300337679| +------------------+ ``` --- layout: model title: Part of Speech for Hindi author: John Snow Labs name: pos_ud_hdtb date: 2020-07-29 23:34:00 +0800 task: Part of Speech Tagging language: hi edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [pos, hi] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_hdtb_hi_2.5.5_2.4_1596054066666.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_hdtb_hi_2.5.5_2.4_1596054066666.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_hdtb", "hi") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_hdtb", "hi") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।"""] pos_df = nlu.load('hi.pos').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='PROPN', metadata={'word': 'उत्तर'}), Row(annotatorType='pos', begin=6, end=7, result='ADP', metadata={'word': 'के'}), Row(annotatorType='pos', begin=9, end=12, result='NOUN', metadata={'word': 'राजा'}), Row(annotatorType='pos', begin=14, end=17, result='VERB', metadata={'word': 'होने'}), Row(annotatorType='pos', begin=19, end=20, result='ADP', metadata={'word': 'के'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_hdtb| |Type:|pos| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|hi| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Turkish BERT Base Uncased (BERTurk) author: John Snow Labs name: bert_base_turkish_uncased date: 2021-05-20 tags: [open_source, embeddings, bert, turkish, tr] task: Embeddings language: tr edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BERTurk is a community-driven cased BERT model for Turkish. Some datasets used for pretraining and evaluation are contributed from the awesome Turkish NLP community, as well as the decision for the model name: BERTurk. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_turkish_uncased_tr_3.1.0_2.4_1621510523359.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_turkish_uncased_tr_3.1.0_2.4_1621510523359.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_turkish_uncased", "tr") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_turkish_uncased", "tr") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("tr.embed.bert.uncased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_turkish_uncased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|tr| |Case sensitive:|true| ## Data Source [https://huggingface.co/dbmdz/bert-base-turkish-uncased](https://huggingface.co/dbmdz/bert-base-turkish-uncased) ## Benchmarking ```bash For results on PoS tagging or NER tasks, please refer to [this repository](https://github.com/stefan-it/turkish-bert). ``` --- layout: model title: RE Pipeline between Body Parts and Procedures author: John Snow Labs name: re_bodypart_proceduretest_pipeline date: 2023-06-13 tags: [licensed, clinical, relation_extraction, body_part, procedures, en] task: Relation Extraction language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_bodypart_proceduretest](https://nlp.johnsnowlabs.com/2021/01/18/re_bodypart_proceduretest_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_bodypart_proceduretest_pipeline_en_4.4.4_3.2_1686664541054.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_bodypart_proceduretest_pipeline_en_4.4.4_3.2_1686664541054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_bodypart_proceduretest_pipeline", "en", "clinical/models") pipeline.fullAnnotate("TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_bodypart_proceduretest_pipeline", "en", "clinical/models") pipeline.fullAnnotate("TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.bodypart_proceduretest.pipeline").predict("""TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.""") ```

## Results ```bash Results | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|------------------------------|---------------|-------------|--------|---------|-------------|-------------|---------------------|------------| | 0 | 1 | External_body_part_or_region | 94 | 98 | chest | Test | 117 | 135 | portable ultrasound | 1.0 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_bodypart_proceduretest_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: Sentence Detection in Malayalam Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [ml, sentence_detection, open_source] task: Sentence Detection language: ml edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ml_3.2.0_3.0_1630336657068.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ml_3.2.0_3.0_1630336657068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "ml") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""ഇംഗ്ലീഷ് വായിക്കുന്ന ഖണ്ഡികകളുടെ മികച്ച ഉറവിടം തേടുകയാണോ? നിങ്ങൾ ശരിയായ സ്ഥലത്ത് എത്തിയിരിക്കുന്നു. അടുത്തിടെ നടത്തിയ ഒരു പഠനമനുസരിച്ച്, ഇന്നത്തെ യുവാക്കളിൽ വായനാശീലം അതിവേഗം കുറഞ്ഞുവരികയാണ്. ഒരു നിശ്ചിത സെക്കൻഡിൽ കൂടുതൽ ഒരു ഇംഗ്ലീഷ് വായന ഖണ്ഡികയിൽ ശ്രദ്ധ കേന്ദ്രീകരിക്കാൻ അവർക്ക് കഴിയില്ല! കൂടാതെ, വായന എല്ലാ മത്സര പരീക്ഷകളുടെയും അവിഭാജ്യ ഘടകമായിരുന്നു. അതിനാൽ, നിങ്ങളുടെ വായനാ കഴിവുകൾ എങ്ങനെ മെച്ചപ്പെടുത്താം? ഈ ചോദ്യത്തിനുള്ള ഉത്തരം യഥാർത്ഥത്തിൽ മറ്റൊരു ചോദ്യമാണ്: വായനാ വൈദഗ്ധ്യത്തിന്റെ ഉപയോഗം എന്താണ്? വായനയുടെ പ്രധാന ലക്ഷ്യം 'അർത്ഥവത്താക്കുക' എന്നതാണ്.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "ml") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("ഇംഗ്ലീഷ് വായിക്കുന്ന ഖണ്ഡികകളുടെ മികച്ച ഉറവിടം തേടുകയാണോ? നിങ്ങൾ ശരിയായ സ്ഥലത്ത് എത്തിയിരിക്കുന്നു. അടുത്തിടെ നടത്തിയ ഒരു പഠനമനുസരിച്ച്, ഇന്നത്തെ യുവാക്കളിൽ വായനാശീലം അതിവേഗം കുറഞ്ഞുവരികയാണ്. ഒരു നിശ്ചിത സെക്കൻഡിൽ കൂടുതൽ ഒരു ഇംഗ്ലീഷ് വായന ഖണ്ഡികയിൽ ശ്രദ്ധ കേന്ദ്രീകരിക്കാൻ അവർക്ക് കഴിയില്ല! കൂടാതെ, വായന എല്ലാ മത്സര പരീക്ഷകളുടെയും അവിഭാജ്യ ഘടകമായിരുന്നു. അതിനാൽ, നിങ്ങളുടെ വായനാ കഴിവുകൾ എങ്ങനെ മെച്ചപ്പെടുത്താം? ഈ ചോദ്യത്തിനുള്ള ഉത്തരം യഥാർത്ഥത്തിൽ മറ്റൊരു ചോദ്യമാണ്: വായനാ വൈദഗ്ധ്യത്തിന്റെ ഉപയോഗം എന്താണ്? വായനയുടെ പ്രധാന ലക്ഷ്യം 'അർത്ഥവത്താക്കുക' എന്നതാണ്.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load('ml.sentence_detector').predict("ഇംഗ്ലീഷ് വായിക്കുന്ന ഖണ്ഡികകളുടെ മികച്ച ഉറവിടം തേടുകയാണോ? നിങ്ങൾ ശരിയായ സ്ഥലത്ത് എത്തിയിരിക്കുന്നു. അടുത്തിടെ നടത്തിയ ഒരു പഠനമനുസരിച്ച്, ഇന്നത്തെ യുവാക്കളിൽ വായനാശീലം അതിവേഗം കുറഞ്ഞുവരികയാണ്. ഒരു നിശ്ചിത സെക്കൻഡിൽ കൂടുതൽ ഒരു ഇംഗ്ലീഷ് വായന ഖണ്ഡികയിൽ ശ്രദ്ധ കേന്ദ്രീകരിക്കാൻ അവർക്ക് കഴിയില്ല! കൂടാതെ, വായന എല്ലാ മത്സര പരീക്ഷകളുടെയും അവിഭാജ്യ ഘടകമായിരുന്നു. അതിനാൽ, നിങ്ങളുടെ വായനാ കഴിവുകൾ എങ്ങനെ മെച്ചപ്പെടുത്താം? ഈ ചോദ്യത്തിനുള്ള ഉത്തരം യഥാർത്ഥത്തിൽ മറ്റൊരു ചോദ്യമാണ്: വായനാ വൈദഗ്ധ്യത്തിന്റെ ഉപയോഗം എന്താണ്? വായനയുടെ പ്രധാന ലക്ഷ്യം 'അർത്ഥവത്താക്കുക' എന്നതാണ്.", output_level ='sentence') ```

## Results ```bash +----------------------------------------------------------------------------------------------------+ |result | +----------------------------------------------------------------------------------------------------+ |[ഇംഗ്ലീഷ് വായിക്കുന്ന ഖണ്ഡികകളുടെ മികച്ച ഉറവിടം തേടുകയാണോ?] | |[നിങ്ങൾ ശരിയായ സ്ഥലത്ത് എത്തിയിരിക്കുന്നു.] | |[അടുത്തിടെ നടത്തിയ ഒരു പഠനമനുസരിച്ച്, ഇന്നത്തെ യുവാക്കളിൽ വായനാശീലം അതിവേഗം കുറഞ്ഞുവരികയാണ്.] | |[ഒരു നിശ്ചിത സെക്കൻഡിൽ കൂടുതൽ ഒരു ഇംഗ്ലീഷ് വായന ഖണ്ഡികയിൽ ശ്രദ്ധ കേന്ദ്രീകരിക്കാൻ അവർക്ക് കഴിയില്ല!]| |[കൂടാതെ, വായന എല്ലാ മത്സര പരീക്ഷകളുടെയും അവിഭാജ്യ ഘടകമായിരുന്നു.] | |[അതിനാൽ, നിങ്ങളുടെ വായനാ കഴിവുകൾ എങ്ങനെ മെച്ചപ്പെടുത്താം?] | |[ഈ ചോദ്യത്തിനുള്ള ഉത്തരം യഥാർത്ഥത്തിൽ മറ്റൊരു ചോദ്യമാണ്:] | |[വായനാ വൈദഗ്ധ്യത്തിന്റെ ഉപയോഗം എന്താണ്?] | |[വായനയുടെ പ്രധാന ലക്ഷ്യം 'അർത്ഥവത്താക്കുക' എന്നതാണ്.] | +----------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|ml| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English BertForQuestionAnswering model (from kaporter) author: John Snow Labs name: bert_qa_kaporter_bert_base_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad` is a English model orginally trained by `kaporter`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_kaporter_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181111131.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_kaporter_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181111131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_kaporter_bert_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_kaporter_bert_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_kaporter").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_kaporter_bert_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kaporter/bert-base-uncased-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0_en_4.0.0_3.0_1655731053643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0_en_4.0.0_3.0_1655731053643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_128d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|422.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-0 --- layout: model title: TREC(6) Question Classifier author: John Snow Labs name: classifierdl_use_trec6 date: 2021-01-08 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [classifier, open_source, en, text_classification] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify open-domain, fact-based questions into one of the following broad semantic categories: Abbreviation, Description, Entities, Human Beings, Locations, or Numeric Values. ## Predicted Entities ``ABBR``, ``DESC``, ``NUM``, ``ENTY``, ``LOC``, ``HUM``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_EN_TREC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_EN_TREC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec6_en_2.7.1_2.4_1610118062425.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec6_en_2.7.1_2.4_1610118062425.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained(lang="en") \ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") document_classifier = ClassifierDLModel.pretrained('classifierdl_use_trec6', 'en') \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") nlpPipeline = Pipeline(stages=[documentAssembler, use, document_classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate('When did the construction of stone circles begin in the UK?') ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val use = UniversalSentenceEncoder.pretrained(lang="en") .setInputCols(Array("document")) .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_use_trec6", "en") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, use, document_classifier)) val data = Seq("When did the construction of stone circles begin in the UK?").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""When did the construction of stone circles begin in the UK?"""] trec6_df = nlu.load('en.classify.trec6.use').predict(text, output_level='document') trec6_df[["document", "trec6"]] ```

## Results ```bash +------------------------------------------------------------------------------------------------+------------+ |document |class | +------------------------------------------------------------------------------------------------+------------+ |When did the construction of stone circles begin in the UK? | NUM | +------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_trec6| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| ## Benchmarking ```bash precision recall f1-score support ABBR 0.00 0.00 0.00 26 DESC 0.89 0.96 0.92 343 ENTY 0.86 0.86 0.86 391 HUM 0.91 0.90 0.91 366 LOC 0.88 0.91 0.89 233 NUM 0.94 0.94 0.94 274 accuracy 0.89 1633 macro avg 0.75 0.76 0.75 1633 weighted avg 0.88 0.89 0.89 1633 ``` ## Data Source This model is trained on the 50 class version of the TREC dataset. http://search.r-project.org/library/textdata/html/dataset_trec.html --- layout: model title: English RobertaForQuestionAnswering Cased model (from AlirezaBaneshi) author: John Snow Labs name: roberta_qa_autotrain_test2_756523214 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-test2-756523214` is a English model originally trained by `AlirezaBaneshi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523214_en_4.3.0_3.0_1674209197246.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523214_en_4.3.0_3.0_1674209197246.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523214","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523214","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_autotrain_test2_756523214| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|415.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AlirezaBaneshi/autotrain-test2-756523214 --- layout: model title: Legal NER in Greek Legislations author: John Snow Labs name: legner_greek_legislation date: 2023-04-25 tags: [el, legal, ner, licensed, legislation] task: Named Entity Recognition language: el edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Legal NER model extracts the following entities from the Greek legislations: - `FACILITY`: Facilities, such as police stations, departments, etc. - `GPE`: Geopolitical Entity; any reference to a geopolitical entity (e.g., country, city, Greek administrative unit, etc.) - `LEG_REF`: Legislation Reference; any reference to Greek or European legislation - `ORG`: Organization; any reference to a public or private organization - `PER`: Any formal name of a person mentioned in the text - `PUBLIC_DOC`: Public Document Reference ## Predicted Entities `FACILITY`, `GPE`, `LEG_REF`, `PUBLIC_DOC`, `PER`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_greek_legislation_el_1.0.0_3.0_1682420832367.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_greek_legislation_el_1.0.0_3.0_1682420832367.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------+----------+ |chunk |ner_label | +----------------------------------------+----------+ |ν. 3148/2003 |LEG_REF | |ΕΥΔΕ/ΕΣΕΑ |ORG | |Δήμου Κυθήρων |GPE | |αριθ. 117/Σ.10η/25 Ιουλ 2016 γνωμοδότηση|PUBLIC_DOC| +----------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_greek_legislation| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|el| |Size:|16.4 MB| ## References In-house annotations ## Benchmarking ```bash label precision recall f1-score support FACILITY 0.94 0.80 0.86 64 GPE 0.77 0.83 0.80 136 LEG_REF 0.94 0.90 0.92 93 ORG 0.85 0.74 0.79 173 PER 0.72 0.71 0.71 58 PUBLIC_DOC 0.76 0.82 0.79 39 micro-avg 0.83 0.80 0.81 563 macro-avg 0.83 0.80 0.81 563 weighted-avg 0.84 0.80 0.82 563 ``` --- layout: model title: Relation Extraction between Biomarkers and Results (ReDL) author: John Snow Labs name: redl_oncology_biomarker_result_biobert_wip date: 2023-01-15 tags: [licensed, clinical, oncology, en, relation_extraction, test, biomarker, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links Biomarker and Oncogene extractions to their corresponding Biomarker_Result extractions. ## Predicted Entities `is_finding_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biomarker_result_biobert_wip_en_4.2.4_3.0_1673766618517.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biomarker_result_biobert_wip_en_4.2.4_3.0_1673766618517.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash +-------------+----------------+-------------+-----------+--------+----------------+-------------+-----------+--------------------+----------+ | relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +-------------+----------------+-------------+-----------+--------+----------------+-------------+-----------+--------------------+----------+ |is_finding_of|Biomarker_Result| 25| 32|negative| Biomarker| 38| 67|thyroid transcrip...|0.99808085| |is_finding_of|Biomarker_Result| 25| 32|negative| Biomarker| 73| 78| napsin|0.99637383| |is_finding_of|Biomarker_Result| 96| 103|positive| Biomarker| 109| 110| ER|0.99221414| |is_finding_of|Biomarker_Result| 96| 103|positive| Biomarker| 116| 117| PR| 0.9893672| | O|Biomarker_Result| 96| 103|positive| Oncogene| 137| 140| HER2| 0.9986272| | O| Biomarker| 109| 110| ER|Biomarker_Result| 124| 131| negative| 0.9999089| | O| Biomarker| 116| 117| PR|Biomarker_Result| 124| 131| negative| 0.9998932| |is_finding_of|Biomarker_Result| 124| 131|negative| Oncogene| 137| 140| HER2|0.98810333| +-------------+----------------+-------------+-----------+--------+----------------+-------------+-----------+--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_biomarker_result_biobert_wip| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.93 0.97 0.95 is_finding_of 0.97 0.93 0.95 macro-avg 0.95 0.95 0.95 ``` --- layout: model title: German RobertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: roberta_qa_ADDI_DE_RoBERTa date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-DE-RoBERTa` is a German model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_DE_RoBERTa_de_4.0.0_3.0_1655726326883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_DE_RoBERTa_de_4.0.0_3.0_1655726326883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ADDI_DE_RoBERTa","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ADDI_DE_RoBERTa","de") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.roberta.de_tuned.by_Gantenbein").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ADDI_DE_RoBERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|de| |Size:|422.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-DE-RoBERTa --- layout: model title: Detect Adverse Drug Events (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_ade date: 2022-01-04 tags: [ner, bertfortokenclassification, adverse, ade, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.0 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse reactions of drugs in reviews, tweets, and medical text using the pretrained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `DRUG`, `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_en_3.4.0_2.4_1641283944065.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_en_3.4.0_2.4_1641283944065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indonesian_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|id| |Size:|468.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/flax-community/indonesian-roberta-base - https://arxiv.org/abs/1907.11692 - https://hf.co/w11wo - https://hf.co/stevenlimcorn - https://hf.co/munggok - https://hf.co/chewkokwah --- layout: model title: Pipeline to Detect clinical entities (ner_healthcare_slim) author: John Snow Labs name: ner_healthcare_slim_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_healthcare_slim](https://nlp.johnsnowlabs.com/2021/04/01/ner_healthcare_slim_de.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_slim_pipeline_de_4.3.0_3.2_1678879973742.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_slim_pipeline_de_4.3.0_3.2_1678879973742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_healthcare_slim_pipeline", "de", "clinical/models") text = '''Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist Hernia femoralis, Akne, einseitig, ein hochmalignes bronchogenes Karzinom, das überwiegend im Zentrum der Lunge, in einem Hauptbronchus entsteht. Die mittlere Prävalenz wird auf 1/20.000 geschätzt.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_healthcare_slim_pipeline", "de", "clinical/models") val text = "Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist Hernia femoralis, Akne, einseitig, ein hochmalignes bronchogenes Karzinom, das überwiegend im Zentrum der Lunge, in einem Hauptbronchus entsteht. Die mittlere Prävalenz wird auf 1/20.000 geschätzt." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------------------|--------:|------:|:------------------|-------------:| | 0 | Bronchialkarzinom | 17 | 33 | MEDICAL_CONDITION | 0.9988 | | 1 | Lungenkrebs | 50 | 60 | MEDICAL_CONDITION | 0.9931 | | 2 | SCLC | 63 | 66 | MEDICAL_CONDITION | 0.9957 | | 3 | Hernia | 73 | 78 | MEDICAL_CONDITION | 0.8134 | | 4 | femoralis | 80 | 88 | BODY_PART | 0.8001 | | 5 | Akne | 91 | 94 | MEDICAL_CONDITION | 0.9678 | | 6 | hochmalignes bronchogenes Karzinom | 112 | 145 | MEDICAL_CONDITION | 0.6409 | | 7 | Lunge | 179 | 183 | BODY_PART | 0.9729 | | 8 | Hauptbronchus | 195 | 207 | BODY_PART | 0.9987 | | 9 | Prävalenz | 232 | 240 | MEDICAL_CONDITION | 0.9986 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare_slim_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|de| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Word2Vec Embeddings in Croatian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, hr, open_source] task: Embeddings language: hr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_hr_3.4.1_3.0_1647292091930.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_hr_3.4.1_3.0_1647292091930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_ptrsxu_chinese_wwm_ext","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_ptrsxu_chinese_wwm_ext","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_ptrsxu_chinese_wwm_ext| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/ptrsxu/chinese-bert-wwm-ext - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Smaller BERT Embeddings (L-10_H-256_A-4) author: John Snow Labs name: small_bert_L10_256 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L10_256_en_2.6.0_2.4_1598344485022.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L10_256_en_2.6.0_2.4_1598344485022.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L10_256", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L10_256", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L10_256').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L10_256_embeddings I [0.14484411478042603, -0.8349236249923706, -1.... love [-0.7449802160263062, -0.4852253794670105, -0.... NLP [-0.03900821506977081, -0.044783130288124084, ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L10_256| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1 --- layout: model title: Bert Embeddings Romanian (Base Cased) author: John Snow Labs name: bert_base_cased date: 2021-09-13 tags: [open_source, embeddings, ro] task: Embeddings language: ro edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus in Romanian Language. The details are described in the paper “BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding”. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_ro_3.2.0_3.0_1631533635237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_ro_3.2.0_3.0_1631533635237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_base_cased", "ro") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_base_cased", "ro") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash Generates 768 dimensional embeddings vectors per token ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_cased| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ro| |Case sensitive:|true| ## Benchmarking ```bash This model is imported from https://huggingface.co/dumitrescustefan/bert-base-romanian-cased-v1 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from monakth) author: John Snow Labs name: distilbert_qa_monakth_base_cased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-finetuned-squad` is a English model originally trained by `monakth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_monakth_base_cased_finetuned_squad_en_4.3.0_3.0_1672766954155.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_monakth_base_cased_finetuned_squad_en_4.3.0_3.0_1672766954155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_monakth_base_cased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_monakth_base_cased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_monakth_base_cased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/monakth/distilbert-base-cased-finetuned-squad --- layout: model title: English XlmRoBertaForQuestionAnswering (from airesearch) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_finetune_qa date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetune-qa` is a English model originally trained by `airesearch`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_finetune_qa_en_4.0.0_3.0_1655989494932.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_finetune_qa_en_4.0.0_3.0_1655989494932.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_finetune_qa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_finetune_qa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_finetune_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|864.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/airesearch/xlm-roberta-base-finetune-qa - https://wandb.ai/cstorm125/wangchanberta-qa - https://github.com/vistec-AI/thai2transformers/blob/dev/scripts/downstream/train_question_answering_lm_finetuning.py --- layout: model title: Detect Cellular/Molecular Biology Entities (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_cellular date: 2021-11-03 tags: [bertfortokenclassification, ner, cellular, en, clinical, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects molecular biology-related terms in medical texts. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `DNA`, `Cell_type`, `Cell_line`, `RNA`, `Protein` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_en_3.3.0_2.4_1635938889847.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_en_3.3.0_2.4_1635938889847.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------------+---------+ |chunk |ner_label| +-------------------------------------------+---------+ |intracellular signaling proteins |protein | |human T-cell leukemia virus type 1 promoter|DNA | |Tax |protein | |Tax-responsive element 1 |DNA | |cyclic AMP-responsive members |protein | |CREB/ATF family |protein | |transcription factors |protein | |Tax |protein | |human T-cell leukemia virus type 1 |DNA | |Tax-responsive element 1 |DNA | |TRE-1 |DNA | |lacZ gene |DNA | |CYC1 promoter |DNA | |TRE-1 |DNA | |cyclic AMP response element-binding protein|protein | |CREB |protein | |CREB |protein | |GAL4 activation domain |protein | |GAD |protein | |reporter gene |DNA | |Tax |protein | +-------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_cellular| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|512| ## Data Source Trained on the JNLPBA corpus containing more than 2.404 publication abstracts. http://www.geniaproject.org/ ## Benchmarking ```bash label precision recall f1-score support B-DNA 0.87 0.77 0.82 1056 B-RNA 0.85 0.79 0.82 118 B-cell_line 0.66 0.70 0.68 500 B-cell_type 0.87 0.75 0.81 1921 B-protein 0.90 0.85 0.88 5067 I-DNA 0.93 0.86 0.90 1789 I-RNA 0.92 0.84 0.88 187 I-cell_line 0.67 0.76 0.71 989 I-cell_type 0.92 0.76 0.84 2991 I-protein 0.94 0.80 0.87 4774 accuracy - - 0.80 19392 macro-avg 0.76 0.81 0.78 19392 weighted-avg 0.89 0.80 0.85 19392 ``` --- layout: model title: English DistilBertForQuestionAnswering Cased model (from shwetha) author: John Snow Labs name: distilbert_qa_autotrain_user_954831770 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-qa-user-954831770` is a English model originally trained by `shwetha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_autotrain_user_954831770_en_4.3.0_3.0_1672765643101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_autotrain_user_954831770_en_4.3.0_3.0_1672765643101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_autotrain_user_954831770","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_autotrain_user_954831770","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_autotrain_user_954831770| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/shwetha/autotrain-qa-user-954831770 --- layout: model title: Swedish asr_lm_swedish TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: pipeline_asr_lm_swedish date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_lm_swedish` is a Swedish model originally trained by birgermoell. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_lm_swedish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_lm_swedish_sv_4.2.0_3.0_1664117937565.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_lm_swedish_sv_4.2.0_3.0_1664117937565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_lm_swedish', lang = 'sv') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_lm_swedish", lang = "sv") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_lm_swedish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|sv| |Size:|757.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nl24 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nl24` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl24_en_4.3.0_3.0_1675123867646.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl24_en_4.3.0_3.0_1675123867646.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nl24","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nl24","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nl24| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|117.0 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nl24 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: French CamemBert Embeddings (from Leisa) author: John Snow Labs name: camembert_embeddings_Leisa_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Leisa`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Leisa_generic_model_fr_3.4.4_3.0_1653986639094.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Leisa_generic_model_fr_3.4.4_3.0_1653986639094.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Leisa_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Leisa/dummy-model --- layout: model title: Fast and Accurate Language Identification - 231 Languages (CNN) author: John Snow Labs name: ld_wiki_cnn_231 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The model is trained on Wikipedia dataset with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Achinese`, `Afrikaans`, `Tosk Albanian`, `Amharic`, `Aragonese`, `Old English`, `Arabic`, `Egyptian Arabic`, `Assamese`, `Asturian`, `Avaric`, `Aymara`, `Azerbaijani`, `South Azerbaijani`, `Bashkir`, `Bavarian`, `bat-smg`, `Central Bikol`, `Belarusian`, `Bulgarian`, `bh`, `Banjar`, `Bengali`, `Tibetan`, `Bishnupriya`, `Breton`, `Bosnian`, `Russia Buriat`, `Catalan`, `cbk-zam`, `Min Dong Chinese`, `Chechen`, `Cebuano`, `Central Kurdish (Soranî)`, `Corsican`, `Crimean Tatar`, `Czech`, `Kashubian`, `Chuvash`, `Welsh`, `Danish`, `German`, `Dimli (individual language)`, `Lower Sorbian`, `dty`, `Dhivehi`, `Greek`, `eml`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Extremaduran`, `Persian`, `Finnish`, `fiu-vro`, `Faroese`, `French`, `Arpitan`, `Friulian`, `Frisian`, `Irish`, `Gagauz`, `Scottish Gaelic`, `Galician`, `Gilaki`, `Guarani`, `Konkani (Goan)`, `Gujarati`, `Manx`, `Hausa`, `Hakka Chinese`, `Hebrew`, `Hindi`, `Fiji Hindi`, `Croatian`, `Upper Sorbian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Interlingue`, `Igbo`, `Ilocano`, `Ido`, `Icelandic`, `Italian`, `Japanese`, `Jamaican Patois`, `Lojban`, `Javanese`, `Georgian`, `Karakalpak`, `Kabyle`, `Kabardian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Komi-Permyak`, `Karachay-Balkar`, `Kölsch`, `Kurdish`, `Komi`, `Cornish`, `Kyrgyz`, `Latin`, `Ladino`, `Luxembourgish`, `Lezghian`, `Luganda`, `Limburgan`, `Ligurian`, `Lombard`, `Lingala`, `Lao`, `Northern Luri`, `Lithuanian`, `Latgalian`, `Latvian`, `Maithili`, `map-bms`, `Moksha`, `Malagasy`, `Meadow Mari`, `Maori`, `Minangkabau`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Hill Mari`, `Malay`, `Maltese`, `Mirandese`, `Burmese`, `Erzya`, `Mazanderani`, `Nahuatl`, `Neapolitan`, `Low German (Low Saxon)`, `nds-nl`, `Nepali`, `Newari`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Narom`, `Pedi`, `Navajo`, `Occitan`, `Livvi`, `Oromo`, `Odia (Oriya)`, `Ossetian`, `Punjabi (Eastern)`, `Pangasinan`, `Kapampangan`, `Papiamento`, `Picard`, `Pennsylvania German`, `Palatine German`, `Polish`, `Punjabi (Western)`, `Pashto`, `Portuguese`, `Quechua`, `Romansh`, `Romanian`, `roa-rup`, `roa-tara`, `Russian`, `Rusyn`, `Kinyarwanda`, `Sanskrit`, `Yakut`, `Sardinian`, `Sicilian`, `Sindhi`, `Northern Sami`, `Serbo-Croatian`, `Sinhala`, `Slovak`, `Slovenian`, `Shona`, `Somali`, `Albanian`, `Serbian`, `Sranan Tongo`, `Saterland Frisian`, `Sundanese`, `Swedish`, `Swahili`, `Silesian`, `Tamil`, `Tulu`, `Telugu`, `Tetun`, `Tajik`, `Thai`, `Turkmen`, `Tagalog`, `Setswana`, `Tongan`, `Turkish`, `Tatar`, `Tuvinian`, `Udmurt`, `Uyghur`, `Ukrainian`, `Urdu`, `Uzbek`, `Venetian`, `Veps`, `Vietnamese`, `Vlaams`, `Volapük`, `Walloon`, `Waray`, `Wolof`, `Shanghainese`, `Xhosa`, `Mingrelian`, `Yiddish`, `Yoruba`, `Zeeuws`, `Chinese`, `zh-classical`, `zh-min-nan`, `zh-yue`. ## Predicted Entities `ace`, `af`, `als`, `am`, `an`, `ang`, `ar`, `arz`, `as`, `ast`, `av`, `ay`, `az`, `azb`, `ba`, `bar`, `bat-smg`, `bcl`, `be`, `bg`, `bh`, `bjn`, `bn`, `bo`, `bpy`, `br`, `bs`, `bxr`, `ca`, `cbk-zam`, `cdo`, `ce`, `ceb`, `ckb`, `co`, `crh`, `cs`, `csb`, `cv`, `cy`, `da`, `de`, `diq`, `dsb`, `dty`, `dv`, `el`, `eml`, `en`, `eo`, `es`, `et`, `ext`, `fa`, `fi`, `fiu-vro`, `fo`, `fr`, `frp`, `fur`, `fy`, `ga`, `gag`, `gd`, `gl`, `glk`, `gn`, `gom`, `gu`, `gv`, `ha`, `hak`, `he`, `hi`, `hif`, `hr`, `hsb`, `ht`, `hu`, `hy`, `ia`, `id`, `ie`, `ig`, `ilo`, `io`, `is`, `it`, `ja`, `jam`, `jbo`, `jv`, `ka`, `kaa`, `kab`, `kbd`, `kk`, `km`, `kn`, `ko`, `koi`, `krc`, `ksh`, `ku`, `kv`, `kw`, `ky`, `la`, `lad`, `lb`, `lez`, `lg`, `li`, `lij`, `lmo`, `ln`, `lo`, `lrc`, `lt`, `ltg`, `lv`, `mai`, `map-bms`, `mdf`, `mg`, `mhr`, `mi`, `min`, `mk`, `ml`, `mn`, `mr`, `mrj`, `ms`, `mt`, `mwl`, `my`, `myv`, `mzn`, `nah`, `nap`, `nds`, `nds-nl`, `ne`, `new`, `nl`, `nn`, `no`, `nrm`, `nso`, `nv`, `oc`, `olo`, `om`, `or`, `os`, `pa`, `pag`, `pam`, `pap`, `pcd`, `pdc`, `pfl`, `pl`, `pnb`, `ps`, `pt`, `qu`, `rm`, `ro`, `roa-rup`, `roa-tara`, `ru`, `rue`, `rw`, `sa`, `sah`, `sc`, `scn`, `sd`, `se`, `sh`, `si`, `sk`, `sl`, `sn`, `so`, `sq`, `sr`, `srn`, `stq`, `su`, `sv`, `sw`, `szl`, `ta`, `tcy`, `te`, `tet`, `tg`, `th`, `tk`, `tl`, `tn`, `to`, `tr`, `tt`, `tyv`, `udm`, `ug`, `uk`, `ur`, `uz`, `vec`, `vep`, `vi`, `vls`, `vo`, `wa`, `war`, `wo`, `wuu`, `xh`, `xmf`, `yi`, `yo`, `zea`, `zh`, `zh-classical`, `zh-min-nan`, `zh-yue`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_cnn_231_xx_2.7.0_2.4_1607183625658.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_cnn_231_xx_2.7.0_2.4_1607183625658.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_cnn_231", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_cnn_231", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_231').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_cnn_231| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | fr| 1000| 996| 0.996| | fi| 1000| 995| 0.995| | sv| 1000| 994| 0.994| | en| 1000| 991| 0.991| | pt| 1000| 988| 0.988| | de| 1000| 986| 0.986| | it| 1000| 982| 0.982| | es| 1000| 977| 0.977| | nl| 1000| 974| 0.974| | lt| 1000| 969| 0.969| | hu| 880| 850|0.9659090909090909| | lv| 916| 884|0.9650655021834061| | el| 1000| 965| 0.965| | pl| 914| 882|0.9649890590809628| | cs| 1000| 964| 0.964| | da| 1000| 963| 0.963| | et| 928| 892|0.9612068965517241| | bg| 1000| 954| 0.954| | sk| 1000| 945| 0.945| | ro| 784| 738|0.9413265306122449| | sl| 914| 850|0.9299781181619255| +--------+-----+-------+------------------+ +-------+--------------------+ |summary| precision| +-------+--------------------+ | count| 21| | mean| 0.9700702474999693| | stddev|0.018256955176991118| | min| 0.9299781181619255| | max| 0.996| +-------+--------------------+ ``` --- layout: model title: English image_classifier_vit_upside_down_classifier ViTForImageClassification from daveni author: John Snow Labs name: image_classifier_vit_upside_down_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_upside_down_classifier` is a English model originally trained by daveni. ## Predicted Entities `original`, `upside_down` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_upside_down_classifier_en_4.1.0_3.0_1660166128533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_upside_down_classifier_en_4.1.0_3.0_1660166128533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_upside_down_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_upside_down_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_upside_down_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Telugu BertForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: bert_embeddings_indic_transformers date: 2022-12-02 tags: [te, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: te edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-te-bert` is a Telugu model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_te_4.2.4_3.0_1670022427927.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_te_4.2.4_3.0_1670022427927.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","te") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","te") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_facility","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_facility","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_facility| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-facility --- layout: model title: Named Entity Recognition Profiling (Clinical) author: John Snow Labs name: ner_profiling_clinical date: 2023-05-04 tags: [licensed, en, clinical, profiling, ner_profiling, ner] task: [Named Entity Recognition, Pipeline Healthcare] language: en edition: Healthcare NLP 4.4.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `embeddings_clinical`. It has been updated by adding new clinical NER models and NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `jsl_ner_wip_clinical`,`jsl_ner_wip_greedy_clinical`,`jsl_ner_wip_modifier_clinical`, `jsl_rd_ner_wip_greedy_clinical`, `ner_abbreviation_clinical`, `ner_ade_binary`, `ner_ade_clinical`, `ner_anatomy`, `ner_anatomy_coarse`, `ner_bacterial_species`, `ner_biomarker`, `ner_biomedical_bc2gm`, `ner_bionlp`, `ner_cancer_genetics`, `ner_cellular`, `ner_chemd_clinical`, `ner_chemicals`, `ner_chemprot_clinical`, `ner_chexpert`, `ner_clinical`, `ner_clinical_large`, `ner_clinical_trials_abstracts`, `ner_covid_trials`, `ner_deid_augmented`, `ner_deid_enriched`, `ner_deid_generic_augmented`,`ner_deid_large`, `ner_deid_sd`,`ner_deid_sd_large`,`ner_deid_subentity_augmented`,`ner_deid_subentity_augmented_i2b2`, `ner_deid_synthetic`, `ner_diseases`, `ner_diseases_large`, `ner_drugprot_clinical`, `ner_drugs`, `ner_drugs_greedy`, `ner_drugs_large`, `ner_eu_clinical_case`, `ner_eu_clinical_condition`, `ner_events_admission_clinical`, `ner_events_clinical`, `ner_financial_contract`, `ner_genetic_variants`, `ner_healthcare`, `ner_human_phenotype_gene_clinical`, `ner_human_phenotype_go_clinical`, `ner_jsl`, `ner_jsl_enriched`, `ner_jsl_slim`, `ner_living_species`, `ner_measurements_clinical`, `ner_medmentions_coarse`, `ner_nature_nero_clinical`, `ner_nihss`, `ner_oncology`, `ner_oncology_anatomy_general`, `ner_oncology_anatomy_granular`, `ner_oncology_biomarker`, `ner_oncology_demographics`, `ner_oncology_diagnosis`, `ner_oncology_posology`, `ner_oncology_response_to_treatment`, `ner_oncology_test`, `ner_oncology_therapy`, `ner_oncology_tnm`, `ner_oncology_unspecific_posology`, `ner_oncology_wip`, `ner_pathogen`, `ner_posology`, `ner_posology_experimental`, `ner_posology_greedy`, `ner_posology_large`, `ner_posology_small`, `ner_radiology`, `ner_radiology_wip_clinical`, `ner_risk_factors`, `ner_sdoh_access_to_healthcare_wip`, `ner_sdoh_community_condition_wip`, `ner_sdoh_demographics_wip`, `ner_sdoh_health_behaviours_problems_wip`, `ner_sdoh_income_social_status_wip`, `ner_sdoh_mentions`, `ner_sdoh_slim_wip`, `ner_sdoh_social_environment_wip`, `ner_sdoh_substance_usage_wip`, `ner_sdoh_wip`, `ner_supplement_clinical`, `ner_vop_anatomy_wip`, `ner_vop_clinical_dept_wip`, `ner_vop_demographic_wip`, `ner_vop_problem_reduced_wip`, `ner_vop_problem_wip`, `ner_vop_slim_wip`, `ner_vop_temporal_wip`, `ner_vop_test_wip`, `ner_vop_treatment_wip`, `ner_vop_wip`, `nerdl_tumour_demo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.4.0_3.2_1683225723531.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.4.0_3.2_1683225723531.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline("ner_profiling_clinical", "en", "clinical/models") result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline("ner_profiling_clinical", "en", "clinical/models") val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_clinical").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ```

## Results ```bash ******************** ner_jsl Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('subsequent', 'Modifier'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Diabetes'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Communicable_Disease'), ('obesity', 'Obesity'), ('body mass index', 'Symptom'), ('33.5 kg/m2', 'Weight'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_diseases_large Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('vomiting', 'Disease')] ******************** ner_radiology Model Results ******************** [('gestational diabetes mellitus', 'Disease_Syndrome_Disorder'), ('type two diabetes mellitus', 'Disease_Syndrome_Disorder'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('acute hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Disease_Syndrome_Disorder'), ('body', 'BodyPart'), ('mass index', 'Symptom'), ('BMI', 'Test'), ('33.5', 'Measurements'), ('kg/m2', 'Units'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus', 'PROBLEM'), ('T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'PROBLEM'), ('BMI', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_medmentions_coarse Model Results ******************** [('female', 'Organism_Attribute'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('T2DM', 'Disease_or_Syndrome'), ('HTG-induced pancreatitis', 'Disease_or_Syndrome'), ('associated with', 'Qualitative_Concept'), ('acute hepatitis', 'Disease_or_Syndrome'), ('obesity', 'Disease_or_Syndrome'), ('body mass index', 'Clinical_Attribute'), ('BMI', 'Clinical_Attribute'), ('polyuria', 'Sign_or_Symptom'), ('polydipsia', 'Sign_or_Symptom'), ('poor appetite', 'Sign_or_Symptom'), ('vomiting', 'Sign_or_Symptom')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_clinical| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.1 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: Named Entity Recognition (NER) Model in Norwegian (Norne 6B 300) author: John Snow Labs name: norne_6B_300 date: 2020-05-06 task: Named Entity Recognition language: "no" edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [ner, nn, nb, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Norne is a Named Entity Recognition (or NER) model of Norvegian, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. Norne 6B 300 is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Derived-`DRV`, Product-`PROD`, Geo-political Entities Location-`GPE_LOC`, Geo-political Entities Organization-`GPE-ORG`, Event-`EVT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_NO/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_NO.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/norne_6B_300_no_2.5.0_2.4_1588781290264.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/norne_6B_300_no_2.5.0_2.4_1588781290264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained('glove_6B_300', lang='xx') \ .setInputCols(['document', 'token']) \ .setOutputCol('embeddings') ner_model = NerDLModel.pretrained("norne_6B_300", "no") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([['William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. [ 9] Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella.']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_6B_300", lang="xx") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("norne_6B_300", "no") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. [ 9] Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (født 28. oktober 1955) er en amerikansk forretningsmagnat, programvareutvikler, investor og filantrop. Han er mest kjent som medgründer av Microsoft Corporation. I løpet av sin karriere hos Microsoft hadde Gates stillingene som styreleder, administrerende direktør (CEO), president og sjef programvarearkitekt, samtidig som han var den største individuelle aksjonæren fram til mai 2014. Han er en av de mest kjente gründere og pionerene i mikrodatarevolusjon på 1970- og 1980-tallet. Han er født og oppvokst i Seattle, Washington, og grunnla Microsoft sammen med barndomsvennen Paul Allen i 1975, i Albuquerque, New Mexico; det fortsatte å bli verdens største programvare for datamaskinprogramvare. Gates ledet selskapet som styreleder og administrerende direktør til han gikk av som konsernsjef i januar 2000, men han forble styreleder og ble sjef for programvarearkitekt. I løpet av slutten av 1990-tallet hadde Gates blitt kritisert for sin forretningstaktikk, som har blitt ansett som konkurransedyktig. Denne uttalelsen er opprettholdt av en rekke dommer. I juni 2006 kunngjorde Gates at han skulle gå over til en deltidsrolle hos Microsoft og på heltid ved Bill & Melinda Gates Foundation, den private veldedige stiftelsen som han og kona, Melinda Gates, opprettet i 2000. Han overførte gradvis arbeidsoppgavene sine til Ray Ozzie og Craig Mundie. Han trakk seg som styreleder for Microsoft i februar 2014 og tiltrådte et nytt verv som teknologirådgiver for å støtte den nyutnevnte administrerende direktøren Satya Nadella."""] ner_df = nlu.load('no.ner.norne.glove.6B_300').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates III |PER | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |CEO |ORG | |Seattle |GPE_LOC | |Washington |GPE_LOC | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |GPE_LOC | |New Mexico |GPE_LOC | |Gates |PER | |Gates |PER | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|ORG | |Melinda Gates |PER | |Ray Ozzie |PER | |Craig Mundie |PER | |Microsoft |ORG | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|norne_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.5.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|no| |Case sensitive:|false| {:.h2_title} ## Data Source The detailed information can be found from [https://www.aclweb.org/anthology/2020.lrec-1.559.pdf](https://www.aclweb.org/anthology/2020.lrec-1.559.pdf) --- layout: model title: Clinical Deidentification (Italian) author: John Snow Labs name: clinical_deidentification date: 2023-06-13 tags: [deidentification, pipeline, it, licensed] task: De-identification language: it edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts in Italian. The pipeline can mask and obfuscate the following entities: `DATE`, `AGE`, `SEX`, `PROFESSION`, `ORGANIZATION`, `PHONE`, `E-MAIL`, `ZIP`, `STREET`, `CITY`, `COUNTRY`, `PATIENT`, `DOCTOR`, `HOSPITAL`, `MEDICALRECORD`, `SSN`, `IDNUM`, `ACCOUNT`, `PLATE`, `USERNAME`, `URL`, and `IPADDR`. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_it_4.4.4_3.2_1686664266856.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_it_4.4.4_3.2_1686664266856.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "it", "clinical/models") sample = """RAPPORTO DI RICOVERO NOME: Lodovico Fibonacci CODICE FISCALE: MVANSK92F09W408A INDIRIZZO: Viale Burcardo 7 CITTÀ : Napoli CODICE POSTALE: 80139 DATA DI NASCITA: 03/03/1946 ETÀ: 70 anni SESSO: M EMAIL: lpizzo@tim.it DATA DI AMMISSIONE: 12/12/2016 DOTTORE: Eva Viviani RAPPORTO CLINICO: 70 anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. Bruno Ferrabosco - ASL Napoli 1 Centro, Dipartimento di Endocrinologia e Nutrizione - Stretto Scamarcio 320, 80138 Napoli EMAIL: bferrabosco@poste.it""" result = deid_pipeline.annotate(sample) ``` ```scala val deid_pipeline = new PretrainedPipeline("clinical_deidentification", "it", "clinical/models") sample = "RAPPORTO DI RICOVERO NOME: Lodovico Fibonacci CODICE FISCALE: MVANSK92F09W408A INDIRIZZO: Viale Burcardo 7 CITTÀ : Napoli CODICE POSTALE: 80139 DATA DI NASCITA: 03/03/1946 ETÀ: 70 anni SESSO: M EMAIL: lpizzo@tim.it DATA DI AMMISSIONE: 12/12/2016 DOTTORE: Eva Viviani RAPPORTO CLINICO: 70 anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. Bruno Ferrabosco - ASL Napoli 1 Centro, Dipartimento di Endocrinologia e Nutrizione - Stretto Scamarcio 320, 80138 Napoli EMAIL: bferrabosco@poste.it" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("it.deid.clinical").predict("""RAPPORTO DI RICOVERO NOME: Lodovico Fibonacci CODICE FISCALE: MVANSK92F09W408A INDIRIZZO: Viale Burcardo 7 CITTÀ : Napoli CODICE POSTALE: 80139 DATA DI NASCITA: 03/03/1946 ETÀ: 70 anni SESSO: M EMAIL: lpizzo@tim.it DATA DI AMMISSIONE: 12/12/2016 DOTTORE: Eva Viviani RAPPORTO CLINICO: 70 anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. Bruno Ferrabosco - ASL Napoli 1 Centro, Dipartimento di Endocrinologia e Nutrizione - Stretto Scamarcio 320, 80138 Napoli EMAIL: bferrabosco@poste.it""") ```

## Results ```bash Results Masked with entity labels ------------------------------ RAPPORTO DI RICOVERO NOME: CODICE FISCALE: INDIRIZZO: CITTÀ : CODICE POSTALE: DATA DI NASCITA: ETÀ: anni SESSO: EMAIL: DATA DI AMMISSIONE: DOTTORE: RAPPORTO CLINICO: anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. - , Dipartimento di Endocrinologia e Nutrizione - , EMAIL: Masked with chars ------------------------------ RAPPORTO DI RICOVERO NOME: [****************] CODICE FISCALE: [**************] INDIRIZZO: [**************] CITTÀ : [****] CODICE POSTALE: [***]DATA DI NASCITA: [********] ETÀ: **anni SESSO: * EMAIL: [***********] DATA DI AMMISSIONE: [********] DOTTORE: [*********] RAPPORTO CLINICO: **anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. [**************] - [*****************], Dipartimento di Endocrinologia e Nutrizione - [*******************], [***] [****] EMAIL: [******************] Masked with fixed length chars ------------------------------ RAPPORTO DI RICOVERO NOME: **** CODICE FISCALE: **** INDIRIZZO: **** CITTÀ : **** CODICE POSTALE: ****DATA DI NASCITA: **** ETÀ: **** anni SESSO: **** EMAIL: **** DATA DI AMMISSIONE: **** DOTTORE: **** RAPPORTO CLINICO: **** anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. **** - ****, Dipartimento di Endocrinologia e Nutrizione - ****, **** **** EMAIL: **** Obfuscated ------------------------------ RAPPORTO DI RICOVERO NOME: Scotto-Polani CODICE FISCALE: ECI-QLN77G15L455Y INDIRIZZO: Viale Orlando 808 CITTÀ : Sesto Raimondo CODICE POSTALE: 53581DATA DI NASCITA: 09/03/1946 ETÀ: 5 anni SESSO: U EMAIL: HenryWatson@world.com DATA DI AMMISSIONE: 10/01/2017 DOTTORE: Sig. Fredo Marangoni RAPPORTO CLINICO: 5 anni, pensionato, senza allergie farmacologiche note, che presenta la seguente storia: ex incidente sul lavoro con fratture vertebrali e costali; operato per la malattia di Dupuytren alla mano destra e un bypass ileo-femorale sinistro; diabete di tipo II, ipercolesterolemia e iperuricemia; alcolismo attivo, fuma 20 sigarette/giorno. È stato indirizzato a noi perché ha presentato un'ematuria macroscopica post-evacuazione in un'occasione e una microematuria persistente in seguito, con un'evacuazione normale. L'esame fisico ha mostrato buone condizioni generali, con addome e genitali normali; l'esame digitale rettale era coerente con un adenoma prostatico di grado I/IV. L'analisi delle urine ha mostrato 4 globuli rossi/campo e 0-5 leucociti/campo; il resto del sedimento era normale. L'emocromo è normale; la biochimica ha mostrato una glicemia di 169 mg/dl e trigliceridi 456 mg/dl; la funzione epatica e renale sono normali. PSA di 1,16 ng/ml. INDIRIZZATO A: Dott. Antonio Rusticucci - ASL 7 DI CARBONIA AZIENDA U.S.L. N. 7, Dipartimento di Endocrinologia e Nutrizione - Via Giorgio 0 Appartamento 26, 03461 Sesto Raimondo EMAIL: murat.g@jsl.com {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|it| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ContextualParserModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - RegexMatcherModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding ICD10-CM Codes author: John Snow Labs name: snomed_icd10cm_mapping date: 2022-06-27 tags: [pipeline, snomed, icd10cm, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_icd10cm_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapping_en_3.5.3_3.0_1656363315439.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapping_en_3.5.3_3.0_1656363315439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("snomed_icd10cm_mapping", "en", "clinical/models") result= pipeline.fullAnnotate("128041000119107 292278006 293072005") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= new PretrainedPipeline("snomed_icd10cm_mapping", "en", "clinical/models") val result= pipeline.fullAnnotate("128041000119107 292278006 293072005") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.snomed_to_icd10cm.pipe").predict("""128041000119107 292278006 293072005""") ```

## Results ```bash | | snomed_code | icd10cm_code | |---:|:----------------------------------------|:---------------------------| | 0 | 128041000119107 | 292278006 | 293072005 | K22.70 | T43.595 | T37.1X5 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_icd10cm_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.5 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Translate Lingala to English Pipeline author: John Snow Labs name: translate_ln_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ln, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ln` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ln_en_xx_2.7.0_2.4_1609698622095.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ln_en_xx_2.7.0_2.4_1609698622095.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ln_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ln_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ln.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ln_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Dutch DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_nl_cased date: 2022-04-12 tags: [distilbert, embeddings, nl, open_source] task: Embeddings language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-nl-cased` is a Dutch model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_nl_cased_nl_3.4.2_3.0_1649783996172.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_nl_cased_nl_3.4.2_3.0_1649783996172.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_nl_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|nl| |Size:|229.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-nl-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Legal Agreement and Plan of Reorganization Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_agreement_and_plan_of_reorganization_bert date: 2022-12-06 tags: [en, legal, classification, agreement, plan, reorganizationlicensed, bert, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_agreement_and_plan_of_reorganization_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `agreement-and-plan-of-reorganization` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `agreement-and-plan-of-reorganization`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agreement_and_plan_of_reorganization_bert_en_1.0.0_3.0_1670349241846.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agreement_and_plan_of_reorganization_bert_en_1.0.0_3.0_1670349241846.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_agreement_and_plan_of_reorganization_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[agreement-and-plan-of-reorganization]| |[other]| |[other]| |[agreement-and-plan-of-reorganization]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agreement_and_plan_of_reorganization_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support agreement-and-plan-of-reorganization 1.00 1.00 1.00 31 other 1.00 1.00 1.00 35 accuracy - - 1.00 66 macro-avg 1.00 1.00 1.00 66 weighted-avg 1.00 1.00 1.00 66 ``` --- layout: model title: Arabic BertForMaskedLM Base Cased model (from aubmindlab) author: John Snow Labs name: bert_embeddings_base_arabertv2 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabertv2` is a Arabic model originally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv2_ar_4.2.4_3.0_1670015875947.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv2_ar_4.2.4_3.0_1670015875947.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv2","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv2","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabertv2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|507.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-base-arabertv2 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-base-arabertv2/blob/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6_en_4.3.0_3.0_1674214999722.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6_en_4.3.0_3.0_1674214999722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_256_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|426.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-6 --- layout: model title: English DistilBertForQuestionAnswering model (from huxxx657) author: John Snow Labs name: distilbert_qa_huxxx657_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_huxxx657_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725541025.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_huxxx657_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725541025.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_huxxx657_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_huxxx657_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_huxxx657").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_huxxx657_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Detect Genes and Human Phenotypes (biobert) author: John Snow Labs name: ner_human_phenotype_gene_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_gene_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_human_phenotype_gene_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_pipeline_en_4.3.0_3.2_1679315678860.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_pipeline_en_4.3.0_3.2_1679315678860.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_human_phenotype_gene_biobert_pipeline", "en", "clinical/models") text = '''Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_human_phenotype_gene_biobert_pipeline", "en", "clinical/models") val text = "Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3)." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phenotype_gene_biobert.pipeline").predict("""Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:------------|-------------:| | 0 | type | 29 | 32 | GENE | 0.9977 | | 1 | polyhydramnios | 75 | 88 | HP | 0.9949 | | 2 | polyuria | 91 | 98 | HP | 0.9955 | | 3 | nephrocalcinosis | 101 | 116 | HP | 0.995 | | 4 | hypokalemia | 122 | 132 | HP | 0.9986 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_gene_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Classify first part of agreements (Parties, Agreement type) author: John Snow Labs name: legclf_introduction_clause_cuad date: 2022-11-25 tags: [en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the first part of a legal agreement, were `PARTIES`, `AGREEMENT TYPE` and `ALIASES` or `ROLES` of the parties are described. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `introduction`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_cuad_en_1.0.0_3.0_1669371916085.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_cuad_en_1.0.0_3.0_1669371916085.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_introduction_clause_cuad", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[introduction]| |[other]| |[other]| |[introduction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_introduction_clause_cuad| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support introduction 1.00 0.98 0.99 99 other 0.99 1.00 0.99 151 accuracy - - 0.99 250 macro-avg 0.99 0.99 0.99 250 weighted--avg 0.99 0.99 0.99 250 ``` --- layout: model title: Extract Entities Related to TNM Staging author: John Snow Labs name: ner_oncology_tnm date: 2022-11-24 tags: [licensed, en, clinical, oncology, ner] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts staging information and mentions related to tumors, lymph nodes and metastases. Definitions of Predicted Entities: - `Cancer_Dx`: Mentions of cancer diagnoses (such as "breast cancer") or pathological types that are usually used as synonyms for "cancer" (e.g. "carcinoma"). When anatomical references are present, they are included in the Cancer_Dx extraction. - `Lymph_Node`: Mentions of lymph nodes and pathological findings of the lymph nodes. - `Lymph_Node_Modifier`: Words that refer to a lymph node being abnormal (such as "enlargement"). - `Metastasis`: Terms that indicate a metastatic disease. Anatomical references are not included in these extractions. - `Staging`: Mentions of cancer stage such as "stage 2b" or "T2N1M0". It also includes words such as "in situ", "early-stage" or "advanced". - `Tumor`: All nonspecific terms that may be related to tumors, either malignant or benign (for example: "mass", "tumor", "lesion", or "neoplasm"). - `Tumor_Description`: Information related to tumor characteristics, such as size, presence of invasion, grade and hystological type. ## Predicted Entities `Cancer_Dx`, `Lymph_Node`, `Lymph_Node_Modifier`, `Metastasis`, `Staging`, `Tumor`, `Tumor_Description` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_tnm_en_4.2.2_3.0_1669308699155.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_tnm_en_4.2.2_3.0_1669308699155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:------------------| | metastatic | Metastasis | | breast carcinoma | Cancer_Dx | | T2N1M1 stage IV | Staging | | 4 cm | Tumor_Description | | tumor | Tumor | | grade 2 | Tumor_Description | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_tnm| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.2 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Lymph_Node 570 77 77 647 0.88 0.88 0.88 Staging 232 22 26 258 0.91 0.90 0.91 Lymph_Node_Modifier 30 5 5 35 0.86 0.86 0.86 Tumor_Description 2651 581 490 3141 0.82 0.84 0.83 Tumor 1116 72 141 1257 0.94 0.89 0.91 Metastasis 358 15 12 370 0.96 0.97 0.96 Cancer_Dx 1302 87 92 1394 0.94 0.93 0.94 macro_avg 6259 859 843 7102 0.90 0.90 0.90 micro_avg 6259 859 843 7102 0.88 0.88 0.88 ``` --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-4_H-512_A-8_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185314705.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185314705.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.uncased_4l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_512_A_8_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|107.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-512_A-8_squad2_covid-qna --- layout: model title: English asr_wav2vec2_base_demo_colab_by_thyagosme TFWav2Vec2ForCTC from thyagosme author: John Snow Labs name: asr_wav2vec2_base_demo_colab_by_thyagosme date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_demo_colab_by_thyagosme` is a English model originally trained by thyagosme. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_demo_colab_by_thyagosme_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_demo_colab_by_thyagosme_en_4.2.0_3.0_1664107996154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_demo_colab_by_thyagosme_en_4.2.0_3.0_1664107996154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_demo_colab_by_thyagosme", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_demo_colab_by_thyagosme", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_demo_colab_by_thyagosme| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: Voice of the Patients (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_emb_clinical_medium_wip date: 2023-04-12 tags: [licensed, clinical, en, ner, vop, patient] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.0 spark_version: [3.0, 3.2] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Allergen`, `SubstanceQuantity`, `RaceEthnicity`, `Measurements`, `InjuryOrPoisoning`, `Treatment`, `Modifier`, `TestResult`, `MedicalDevice`, `Vaccine`, `Frequency`, `HealthStatus`, `Route`, `RelationshipStatus`, `Procedure`, `Duration`, `DateTime`, `AdmissionDischarge`, `Disease`, `Test`, `Substance`, `Laterality`, `Symptom`, `ClinicalDept`, `Dosage`, `Age`, `Drug`, `VitalTest`, `PsychologicalCondition`, `Form`, `BodyPart`, `Employment`, `Gender` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/VOICE_OF_THE_PATIENTS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_medium_wip_en_4.4.0_3.0_1681315530573.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_medium_wip_en_4.4.0_3.0_1681315530573.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_amitjohn007_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_amitjohn007_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_amitjohn007_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/amitjohn007/roberta-base-finetuned-squad --- layout: model title: Named Entity Recognition (NER) Model in Danish (Dane 840B 300) author: John Snow Labs name: dane_ner_840B_300 date: 2020-08-30 task: Named Entity Recognition language: da edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [ner, da, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Norne is a Named Entity Recognition (or NER) model of Danish, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. Dane NER 840B 300 is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_DA/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dane_ner_840B_300_da_2.6.0_2.4_1598810268070.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dane_ner_840B_300_da_2.6.0_2.4_1598810268070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates |PER | |amerikansk |MISC | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |1970'erne |MISC | |1980'erne |MISC | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | |1990'erne |MISC | |Gates |MISC | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|PER | |Melinda Gates |PER | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dane_ner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.6.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|da| |Case sensitive:|false| {:.h2_title} ## Data Source The detailed information can be found from [https://www.aclweb.org/anthology/2020.lrec-1.565.pdf](https://www.aclweb.org/anthology/2020.lrec-1.565.pdf) --- layout: model title: Legal Parties in interest Clause Binary Classifier author: John Snow Labs name: legclf_parties_in_interest_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `parties-in-interest` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `parties-in-interest` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_parties_in_interest_clause_en_1.0.0_3.2_1660122826311.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_parties_in_interest_clause_en_1.0.0_3.2_1660122826311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_parties_in_interest_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[parties-in-interest]| |[other]| |[other]| |[parties-in-interest]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_parties_in_interest_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.99 0.99 106 parties-in-interest 0.98 0.98 0.98 49 accuracy - - 0.99 155 macro-avg 0.99 0.99 0.99 155 weighted-avg 0.99 0.99 0.99 155 ``` --- layout: model title: Longformer Base NER Pipeline author: John Snow Labs name: longformer_base_token_classifier_conll03_pipeline date: 2022-06-19 tags: [ner, longformer, pipeline, conll, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [longformer_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/10/09/longformer_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653913352.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655653913352.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_test", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_test", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_test| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| --- layout: model title: Extract treatment entities (Voice of the Patients) author: John Snow Labs name: ner_vop_treatment_wip date: 2023-04-20 tags: [licensed, clinical, en, ner, vop, patient, treatment] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts treatments mentioned in documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Treatment`, `Frequency`, `Procedure`, `Route`, `Duration`, `Dosage`, `Drug`, `Form` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_wip_en_4.4.0_3.0_1682013186202.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_wip_en_4.4.0_3.0_1682013186202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------|:------------| | metformin | Drug | | glipizide | Drug | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_treatment_wip| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.9 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I”m 20 year old girl. I”m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I”m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I”m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I”m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Treatment 144 44 67 211 0.77 0.68 0.72 Frequency 801 111 271 1072 0.88 0.75 0.81 Procedure 505 104 112 617 0.83 0.82 0.82 Route 29 3 8 37 0.91 0.78 0.84 Duration 1926 382 345 2271 0.83 0.85 0.84 Dosage 350 56 49 399 0.86 0.88 0.87 Drug 1210 125 108 1318 0.91 0.92 0.91 Form 235 23 18 253 0.91 0.93 0.92 macro_avg 5200 848 978 6178 0.86 0.83 0.84 micro_avg 5200 848 978 6178 0.86 0.84 0.85 ``` --- layout: model title: Adverse Drug Events Classifier (LogReg) author: John Snow Labs name: classifier_logreg_ade date: 2023-05-11 tags: [text_classification, ade, clinical, licensed, logreg, en] task: Text Classification language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: DocumentLogRegClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with the Logistic Regression algorithm and classifies text/sentence into two categories: True : The sentence is talking about a possible ADE False : The sentence doesn’t have any information about an ADE. The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). ## Predicted Entities `True`, `False` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifier_logreg_ade_en_4.4.1_3.0_1683817451286.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifier_logreg_ade_en_4.4.1_3.0_1683817451286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------+-------+ |text |result | +----------------------------------------------------------------------------------------+-------+ |Detection of activated eosinophils in nasal polyps of an aspirin-induced asthma patient.|[True] | |None of the patients required treatment for the overdose. |[False]| +----------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifier_logreg_ade| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[prediction]| |Language:|en| |Size:|595.7 KB| ## References The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). Reference: Gurulingappa et al., Benchmark Corpus to Support Information Extraction for Adverse Drug Effects, JBI, 2012. http://www.sciencedirect.com/science/article/pii/S1532046412000615 ## Benchmarking ```bash label precision recall f1-score support False 0.91 0.92 0.92 3362 True 0.79 0.79 0.79 1361 accuracy - - 0.88 4723 macro_avg 0.85 0.85 0.85 4723 weighted_avg 0.88 0.88 0.88 4723 ``` --- layout: model title: English asr_english_filipino_wav2vec2_l_xls_r_test_09 TFWav2Vec2ForCTC from Khalsuu author: John Snow Labs name: pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_english_filipino_wav2vec2_l_xls_r_test_09` is a English model originally trained by Khalsuu. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09_en_4.2.0_3.0_1664119422310.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09_en_4.2.0_3.0_1664119422310.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_09| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: German BertForMaskedLM Large Cased model (from deepset) author: John Snow Labs name: bert_embeddings_g_large date: 2022-12-02 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gbert-large` is a German model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_large_de_4.2.4_3.0_1670022204873.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_large_de_4.2.4_3.0_1670022204873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_large","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_large","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_g_large| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gbert-large - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - http://deepset.ai/ - https://haystack.deepset.ai/ - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/haystack - https://docs.haystack.deepset.ai - https://haystack.deepset.ai/community - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_512 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_512_zh_4.2.4_3.0_1670021676308.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_512_zh_4.2.4_3.0_1670021676308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_512| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|90.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-512 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: English BertForQuestionAnswering model (from Vasanth) author: John Snow Labs name: bert_qa_bert_base_uncased_qa_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-qa-squad2` is a English model orginally trained by `Vasanth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_qa_squad2_en_4.0.0_3.0_1654181282352.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_qa_squad2_en_4.0.0_3.0_1654181282352.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_qa_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.base_uncased.by_Vasanth").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Vasanth/bert-base-uncased-qa-squad2 --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk TFWav2Vec2ForCTC from krirk author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk` is a English model originally trained by krirk. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk_en_4.2.0_3.0_1664042673180.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk_en_4.2.0_3.0_1664042673180.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_krirk| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English AlbertForQuestionAnswering model (from rowan1224) author: John Snow Labs name: albert_qa_slp date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `albert-slp` is a English model originally trained by `rowan1224`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_slp_en_4.0.0_3.0_1656063737900.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_slp_en_4.0.0_3.0_1656063737900.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_slp","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_slp","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.albert.by_rowan1224").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_slp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rowan1224/albert-slp --- layout: model title: Fast Neural Machine Translation Model from English to Italic Languages author: John Snow Labs name: opus_mt_en_itc date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, itc, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `itc` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_itc_xx_2.7.0_2.4_1609170676219.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_itc_xx_2.7.0_2.4_1609170676219.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_itc| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Cased model (from saraks) author: John Snow Labs name: distilbert_qa_cuad_parties_08_25 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cuad-distil-parties-08-25` is a English model originally trained by `saraks`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_parties_08_25_en_4.3.0_3.0_1672766229642.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_parties_08_25_en_4.3.0_3.0_1672766229642.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_parties_08_25","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_parties_08_25","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cuad_parties_08_25| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saraks/cuad-distil-parties-08-25 --- layout: model title: Multilingual DistilBertForQuestionAnswering model (from ZYW) Squad author: John Snow Labs name: distilbert_qa_squad_en_de_es_vi_zh_model date: 2022-06-08 tags: [en, de, vi, zh, es, open_source, distilbert, question_answering, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-en-de-es-vi-zh-model` is a English model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_en_de_es_vi_zh_model_xx_4.0.0_3.0_1654728800543.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_en_de_es_vi_zh_model_xx_4.0.0_3.0_1654728800543.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_en_de_es_vi_zh_model","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_en_de_es_vi_zh_model","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.squad.distil_bert._en_de_es_vi_zh_tuned.by_ZYW").predict("""PUT YOUR QUESTION HERE|||"PUT YOUR CONTEXT HERE""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_en_de_es_vi_zh_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-en-de-es-vi-zh-model --- layout: model title: Fast and Accurate Language Identification - 220 Languages (CNN) author: John Snow Labs name: ld_wiki_tatoeba_cnn_220 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The model is trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Achinese`, `Afrikaans`, `Tosk Albanian`, `Amharic`, `Aragonese`, `Old English`, `Arabic`, `Egyptian Arabic`, `Assamese`, `Asturian`, `Avaric`, `Aymara`, `Azerbaijani`, `South Azerbaijani`, `Bashkir`, `Bavarian`, `bat-smg`, `Central Bikol`, `Belarusian`, `Bulgarian`, `bh`, `Bengali`, `Tibetan`, `Bishnupriya`, `Breton`, `Russia Buriat`, `Catalan`, `Min Dong Chinese`, `Chechen`, `Cebuano`, `Central Kurdish (Soranî)`, `Corsican`, `Crimean Tatar`, `Czech`, `Kashubian`, `Chuvash`, `Welsh`, `Danish`, `German`, `Dimli (individual language)`, `Lower Sorbian`, `Dhivehi`, `Greek`, `eml`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Basque`, `Extremaduran`, `Persian`, `Finnish`, `fiu-vro`, `Faroese`, `French`, `Arpitan`, `Friulian`, `Frisian`, `Irish`, `Gagauz`, `Scottish Gaelic`, `Galician`, `Guarani`, `Konkani (Goan)`, `Gujarati`, `Manx`, `Hausa`, `Hakka Chinese`, `Hebrew`, `Hindi`, `Fiji Hindi`, `Upper Sorbian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Interlingue`, `Igbo`, `Ilocano`, `Ido`, `Icelandic`, `Italian`, `Japanese`, `Jamaican Patois`, `Lojban`, `Javanese`, `Georgian`, `Karakalpak`, `Kabyle`, `Kabardian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Komi-Permyak`, `Karachay-Balkar`, `Kölsch`, `Kurdish`, `Komi`, `Cornish`, `Kyrgyz`, `Latin`, `Ladino`, `Luxembourgish`, `Lezghian`, `Luganda`, `Limburgan`, `Ligurian`, `Lombard`, `Lingala`, `Lao`, `Northern Luri`, `Lithuanian`, `Latvian`, `Maithili`, `map-bms`, `Malagasy`, `Meadow Mari`, `Maori`, `Minangkabau`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Hill Mari`, `Maltese`, `Mirandese`, `Burmese`, `Erzya`, `Mazanderani`, `Nahuatl`, `Neapolitan`, `Low German (Low Saxon)`, `nds-nl`, `Nepali`, `Newari`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Narom`, `Pedi`, `Navajo`, `Occitan`, `Livvi`, `Oromo`, `Odia (Oriya)`, `Ossetian`, `Punjabi (Eastern)`, `Pangasinan`, `Kapampangan`, `Papiamento`, `Picard`, `Palatine German`, `Polish`, `Punjabi (Western)`, `Pashto`, `Portuguese`, `Quechua`, `Romansh`, `Romanian`, `roa-tara`, `Russian`, `Rusyn`, `Kinyarwanda`, `Sanskrit`, `Yakut`, `Sardinian`, `Sicilian`, `Scots`, `Sindhi`, `Northern Sami`, `Sinhala`, `Slovak`, `Slovenian`, `Shona`, `Somali`, `Albanian`, `Serbian`, `Saterland Frisian`, `Sundanese`, `Swedish`, `Swahili`, `Silesian`, `Tamil`, `Tulu`, `Telugu`, `Tetun`, `Tajik`, `Thai`, `Turkmen`, `Tagalog`, `Setswana`, `Tongan`, `Turkish`, `Tatar`, `Tuvinian`, `Udmurt`, `Uyghur`, `Ukrainian`, `Urdu`, `Uzbek`, `Venetian`, `Veps`, `Vietnamese`, `Vlaams`, `Volapük`, `Walloon`, `Waray`, `Wolof`, `Shanghainese`, `Xhosa`, `Mingrelian`, `Yiddish`, `Yoruba`, `Zeeuws`, `Chinese`, `zh-classical`, `zh-min-nan`, `zh-yue`. ## Predicted Entities `ace`, `af`, `als`, `am`, `an`, `ang`, `ar`, `arz`, `as`, `ast`, `av`, `ay`, `az`, `azb`, `ba`, `bar`, `bat-smg`, `bcl`, `be`, `bg`, `bh`, `bn`, `bo`, `bpy`, `br`, `bxr`, `ca`, `cdo`, `ce`, `ceb`, `ckb`, `co`, `crh`, `cs`, `csb`, `cv`, `cy`, `da`, `de`, `diq`, `dsb`, `dv`, `el`, `eml`, `en`, `eo`, `es`, `et`, `eu`, `ext`, `fa`, `fi`, `fiu-vro`, `fo`, `fr`, `frp`, `fur`, `fy`, `ga`, `gag`, `gd`, `gl`, `gn`, `gom`, `gu`, `gv`, `ha`, `hak`, `he`, `hi`, `hif`, `hsb`, `ht`, `hu`, `hy`, `ia`, `id`, `ie`, `ig`, `ilo`, `io`, `is`, `it`, `ja`, `jam`, `jbo`, `jv`, `ka`, `kaa`, `kab`, `kbd`, `kk`, `km`, `kn`, `ko`, `koi`, `krc`, `ksh`, `ku`, `kv`, `kw`, `ky`, `la`, `lad`, `lb`, `lez`, `lg`, `li`, `lij`, `lmo`, `ln`, `lo`, `lrc`, `lt`, `lv`, `mai`, `map-bms`, `mg`, `mhr`, `mi`, `min`, `mk`, `ml`, `mn`, `mr`, `mrj`, `mt`, `mwl`, `my`, `myv`, `mzn`, `nah`, `nap`, `nds`, `nds-nl`, `ne`, `new`, `nl`, `nn`, `no`, `nrm`, `nso`, `nv`, `oc`, `olo`, `om`, `or`, `os`, `pa`, `pag`, `pam`, `pap`, `pcd`, `pfl`, `pl`, `pnb`, `ps`, `pt`, `qu`, `rm`, `ro`, `roa-tara`, `ru`, `rue`, `rw`, `sa`, `sah`, `sc`, `scn`, `sco`, `sd`, `se`, `si`, `sk`, `sl`, `sn`, `so`, `sq`, `sr`, `stq`, `su`, `sv`, `sw`, `szl`, `ta`, `tcy`, `te`, `tet`, `tg`, `th`, `tk`, `tl`, `tn`, `to`, `tr`, `tt`, `tyv`, `udm`, `ug`, `uk`, `ur`, `uz`, `vec`, `vep`, `vi`, `vls`, `vo`, `wa`, `war`, `wo`, `wuu`, `xh`, `xmf`, `yi`, `yo`, `zea`, `zh`, `zh-classical`, `zh-min-nan`, `zh-yue`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_220_xx_2.7.0_2.4_1607184539094.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_220_xx_2.7.0_2.4_1607184539094.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_220", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_220", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_220').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_tatoeba_cnn_220| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia and Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | sv| 1000| 999| 0.999| | fr| 1000| 999| 0.999| | fi| 1000| 998| 0.998| | it| 1000| 997| 0.997| | pt| 1000| 995| 0.995| | el| 1000| 994| 0.994| | de| 1000| 993| 0.993| | en| 1000| 990| 0.99| | nl| 1000| 987| 0.987| | hu| 880| 866|0.9840909090909091| | da| 1000| 980| 0.98| | es| 1000| 976| 0.976| | ro| 784| 765|0.9757653061224489| | et| 928| 905|0.9752155172413793| | lt| 1000| 975| 0.975| | cs| 1000| 973| 0.973| | pl| 914| 889|0.9726477024070022| | sk| 1000| 941| 0.941| | bg| 1000| 939| 0.939| | lv| 916| 857|0.9355895196506551| | sl| 914| 789|0.8632385120350109| +--------+-----+-------+------------------+ +-------+-------------------+ |summary| precision| +-------+-------------------+ | count| 21| | mean| 0.9734546412641623| | stddev|0.03176749551086062| | min| 0.8632385120350109| | max| 0.999| +-------+-------------------+ ``` --- layout: model title: Hausa Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_ner_hausa date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, ha, open_source] task: Named Entity Recognition language: ha edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-ner-hausa` is a Hausa model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_hausa_ha_3.4.2_3.0_1652808366927.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_hausa_ha_3.4.2_3.0_1652808366927.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_ner_hausa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ha| |Size:|775.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-ner-hausa - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman/NERTransfer - htt --- layout: model title: English BertForQuestionAnswering Small Cased model (from motiondew) author: John Snow Labs name: bert_qa_sd3_small date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-sd3-small` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sd3_small_en_4.0.0_3.0_1657188265535.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sd3_small_en_4.0.0_3.0_1657188265535.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sd3_small","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_sd3_small","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sd3_small| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-sd3-small --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from drewski) author: John Snow Labs name: distilbert_qa_drewski_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `drewski`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_drewski_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770549448.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_drewski_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770549448.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_drewski_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_drewski_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_drewski_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/drewski/distilbert-base-uncased-finetuned-squad --- layout: model title: English Bert Embeddings (from kornosk) author: John Snow Labs name: bert_embeddings_bert_political_election2020_twitter_mlm date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-political-election2020-twitter-mlm` is a English model orginally trained by `kornosk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_political_election2020_twitter_mlm_en_3.4.2_3.0_1649672268288.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_political_election2020_twitter_mlm_en_3.4.2_3.0_1649672268288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_political_election2020_twitter_mlm| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/kornosk/bert-political-election2020-twitter-mlm - https://www.aclweb.org/anthology/2021.naacl-main.376 - https://github.com/GU-DataLab/stance-detection-KE-MLM - https://www.aclweb.org/anthology/2021.naacl-main.376 --- layout: model title: Typed Dependency Parsing pipeline for English author: John Snow Labs name: dependency_parse date: 2021-03-27 tags: [pipeline, dependency_parsing, untyped_dependency_parsing, typed_dependency_parsing, laballed_depdency_parsing, unlaballed_depdency_parsing, en, open_source] supported: true task: [Dependency Parser, Pipeline Public] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: Pipeline article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Typed Dependency parser, trained on the on the CONLL dataset. Dependency parsing is the task of extracting a dependency parse of a sentence that represents its grammatical structure and defines the relationships between “head” words and words, which modify those heads. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dependency_parse_en_3.0.0_3.0_1616864258046.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dependency_parse_en_3.0.0_3.0_1616864258046.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('dependency_parse', lang = 'en') annotations = pipeline.fullAnnotate("Dependencies represents relationships betweens words in a Sentence "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("dependency_parse", lang = "en") val result = pipeline.fullAnnotate("Dependencies represents relationships betweens words in a Sentence")(0) ``` {:.nlu-block} ```python nlu.load("dep.typed").predict("Dependencies represents relationships betweens words in a Sentence") ```

## Results ```bash +---------------------------------------------------------------------------------+--------------------------------------------------------+ |result |result | +---------------------------------------------------------------------------------+--------------------------------------------------------+ |[ROOT, Dependencies, represents, words, relationships, Sentence, Sentence, words]|[root, parataxis, nsubj, amod, nsubj, case, nsubj, flat]| +---------------------------------------------------------------------------------+--------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dependency_parse| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetector - Tokenizer - PerceptronModel - DependencyParserModel - TypedDependencyParserModel --- layout: model title: Word2Vec Embeddings in Quechua (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, qu, open_source] task: Embeddings language: qu edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_qu_3.4.1_3.0_1647453594212.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_qu_3.4.1_3.0_1647453594212.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+ |result | +-------------------------+ |[Nejste, lepšit, než, já]| +-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|cs| |Size:|379.0 KB| --- layout: model title: Clinical Deidentification (glove) author: John Snow Labs name: clinical_deidentification_glove date: 2023-06-13 tags: [deidentification, en, licensed, pipeline] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with lightweight `glove_100d` embeddings and can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_en_4.4.4_3.2_1686663547185.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_en_4.4.4_3.2_1686663547185.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification_glove", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification_glove","en","clinical/models") val result = deid_pipeline.annotate("Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.glove_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

## Results ```bash Results Masked with entity labels ------------------------------ Name : , Record date: , # . Dr. , ID: , IP . He is a male was admitted to the for cystectomy on . Patient's VIN : , SSN , Driver's license . Phone , , , E-MAIL: . Masked with chars ------------------------------ Name : [**************], Record date: [********], # [****]. Dr. [********], ID: [********], IP [************]. He is a [*********] male was admitted to the [**********] for cystectomy on [******]. Patient's VIN : [***************], SSN [**********], Driver's license [*********]. Phone [************], [***************], [***********], E-MAIL: [*************]. Masked with fixed length chars ------------------------------ Name : ****, Record date: ****, # ****. Dr. ****, ID: ****, IP ****. He is a **** male was admitted to the **** for cystectomy on ****. Patient's VIN : ****, SSN ****, Driver's license ****. Phone ****, ****, ****, E-MAIL: ****. Obfuscated ------------------------------ Name : Berneta Anis, Record date: 2093-02-19, # U4660137. Dr. Dr Worley Colonel, ID: ZJ:9570208, IP 005.005.005.005. He is a 67 male was admitted to the ST. LUKE'S HOSPITAL AT THE VINTAGE for cystectomy on 06-02-1981. Patient's VIN : 3CCCC22DDDD333888, SSN SSN-618-77-1042, Driver's license W693817528998. Phone 0496 46 46 70, 3100 weston rd, Shattuck, E-MAIL: Freddi@hotmail.com. {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification_glove| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|181.4 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Financial English BERT Embeddings (Number shape masking) author: John Snow Labs name: bert_embeddings_sec_bert_sh date: 2022-04-12 tags: [bert, embeddings, en, open_source, financial] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Financial BERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `sec-bert-shape` is a English model orginally trained by `nlpaueb`.This model is the same as Bert Base but we replace numbers with pseudo-tokens that represent the number’s shape, so numeric expressions (of known shapes) are no longer fragmented, e.g., '53.2' becomes '[XX.X]' and '40,200.5' becomes '[XX,XXX.X]'. If you are interested in Financial Embeddings, take a look also at these two models: - [sec-base](https://nlp.johnsnowlabs.com/2022/04/12/bert_embeddings_sec_bert_base_en_3_0.html): Same as BERT Base but trained with financial documents. - [sec-num](https://nlp.johnsnowlabs.com/2022/04/12/bert_embeddings_sec_bert_num_en_3_0.html): Same as Bert sec-base but we replace every number token with a [NUM] pseudo-token handling all numeric expressions in a uniform manner, disallowing their fragmentation). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_sec_bert_sh_en_3.4.2_3.0_1649758845734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_sec_bert_sh_en_3.4.2_3.0_1649758845734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_sec_bert_sh| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/nlpaueb/sec-bert-shape - https://arxiv.org/abs/2203.06482 - http://nlp.cs.aueb.gr/ --- layout: model title: Stopwords Remover for Serbian language (389 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, sr, open_source] task: Stop Words Removal language: sr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_sr_3.4.1_3.0_1646673003296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_sr_3.4.1_3.0_1646673003296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[бољи]| +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|sr| |Size:|2.7 KB| --- layout: model title: Sentence Entity Resolver for LOINC (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_loinc_augmented date: 2021-11-23 tags: [loinc, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.2 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted clinical NER entities to LOINC codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It trained on the augmented version of the dataset which is used in previous LOINC resolver models. ## Predicted Entities {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_augmented_en_3.3.2_2.4_1637664939262.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_augmented_en_3.3.2_2.4_1637664939262.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_loinc_augmented``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Test, BMI, HDL, LDL, Medical_Device, Temperature, Total_Cholesterol, Triglycerides, Blood_Pressure``` set in ```.setWhiteList()```.

## Results ```bash +--------------------------+-----+---+------+----------+----------+--------------------------------------------------+--------------------------------------------------+ | chunk|begin|end|entity|confidence|Loinc_Code| all_codes| resolutions| +--------------------------+-----+---+------+----------+----------+--------------------------------------------------+--------------------------------------------------+ | Body mass index| 74| 88| Test|0.39306664| LP35925-4|LP35925-4:::BDYCRC:::LP172732-2:::39156-5:::LP7...|body mass index:::body circumference:::body mus...| |aspartate aminotransferase| 111|136| Test| 0.74925| LP15426-7|LP15426-7:::14409-7:::LP307348-5:::LP15333-5:::...|aspartate aminotransferase::: aspartate transam...| | alanine aminotransferase| 146|169| Test| 0.9579| LP15333-5|LP15333-5:::LP307326-1:::16324-6:::LP307348-5::...|alanine aminotransferase:::alanine aminotransfe...| | hgba1c| 180|185| Test| 0.1118| 17855-8|17855-8:::4547-6:::55139-0:::72518-4:::45190-6:...| hba1c::: hgb a1::: hb1::: hcds1::: hhc1::: htr...| +--------------------------+-----+---+------+----------+----------+--------------------------------------------------+--------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_loinc_augmented| |Compatibility:|Healthcare NLP 3.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[loinc_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on standard LOINC coding system. --- layout: model title: Generic Deidentification NER (SEC Bert Embeddings) author: John Snow Labs name: finner_deid_sec date: 2023-02-24 tags: [deid, deidentification, anonymization, en, licensed] task: Named Entity Recognition language: en edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model which allows you to detect some generic entities that may require to be masked or obfuscated to be compliant with different regulations, as GDPR and CCPA. This is just an NER model, make sure you try the full De-identification pipelines available in Models Hub. The only difference between this and `finner_deid` is the embeddings used. ## Predicted Entities `AGE`, `CITY`, `COUNTRY`, `DATE`, `EMAIL`, `FAX`, `LOCATION-OTHER`, `ORG`, `PERSON`, `PHONE`, `PROFESSION`, `STATE`, `STREET`, `URL`, `ZIP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/DEID_FIN/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_deid_sec_en_1.0.0_3.0_1677282571388.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_deid_sec_en_1.0.0_3.0_1677282571388.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+----------------+ | token| ner_label| +-----------+----------------+ | This| O| | LICENSE| O| | AND| O| |DEVELOPMENT| O| | AGREEMENT| O| | (| O| | this| O| | Agreement| O| | )| O| | is| O| | entered| O| | into| O| | effective| O| | as| O| | of| O| | Nov| B-DATE| | .| I-DATE| | 02| I-DATE| | ,| I-DATE| | 2019| I-DATE| | (| O| | the| O| | Effective| O| | Date| O| | )| O| | by| O| | and| O| | between| O| | Bioeq| O| | IP| O| | AG| O| | ,| O| | having| O| | its| O| | principal| O| | place| O| | of| O| | business| O| | at| O| | 333| B-STREET| | Twin| I-STREET| | Dolphin| I-STREET| | Drive| I-STREET| | ,| O| | Suite|B-LOCATION-OTHER| | 600|I-LOCATION-OTHER| | ,| O| | Redwood| B-CITY| | City| I-CITY| | ,| O| | CA| B-STATE| | ,| O| | 94065| B-ZIP| | ,| O| | USA| B-STATE| | (| O| | Licensee| O| | ).| O| +-----------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_deid_sec| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.4 MB| ## References In-house annotated documents with protected information ## Benchmarking ```bash label precision recall f1-score support B-AGE 0.96 0.89 0.92 245 B-CITY 0.85 0.86 0.86 123 B-COUNTRY 0.86 0.67 0.75 36 B-DATE 0.98 0.97 0.97 2352 B-ORG 0.75 0.71 0.73 38 B-PERSON 0.97 0.94 0.95 1348 B-PHONE 0.86 0.80 0.83 86 B-PROFESSION 0.93 0.75 0.83 84 B-STATE 0.92 0.89 0.91 102 B-STREET 0.99 0.91 0.95 89 I-CITY 0.82 0.77 0.79 35 I-COUNTRY 1.00 0.50 0.67 6 I-DATE 0.96 0.95 0.96 402 I-ORG 0.71 0.86 0.77 28 I-PERSON 0.98 0.96 0.97 1240 I-PHONE 0.91 0.92 0.92 77 I-PROFESSION 0.96 0.79 0.87 70 I-STATE 1.00 0.62 0.77 8 I-STREET 0.98 0.94 0.96 188 I-ZIP 0.84 0.97 0.90 60 O 1.00 1.00 1.00 194103 accuracy - - 1.00 200762 macro-avg 0.72 0.62 0.65 200762 weighted-avg 1.00 1.00 1.00 200762 ``` --- layout: model title: Fast Neural Machine Translation Model from Italic Languages to English author: John Snow Labs name: opus_mt_itc_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, itc, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `itc` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_itc_en_xx_2.7.0_2.4_1609170571711.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_itc_en_xx_2.7.0_2.4_1609170571711.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_itc_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Navteca's Tapas Table Understanding (Large, WTQ) author: John Snow Labs name: table_qa_tapas_large_finetuned_wtq date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Large Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_large_finetuned_wtq_en_4.2.0_3.0_1664530763103.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_large_finetuned_wtq_en_4.2.0_3.0_1664530763103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_large_finetuned_wtq","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_mini_arabic","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_mini_arabic","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_mini_arabic| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|43.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/asafaya/bert-mini-arabic - https://traces1.inria.fr/oscar/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/bert - https://www.tensorflow.org/tfrc - https://github.com/alisafaya/Arabic-BERT --- layout: model title: Legal United Nations Document Classifier (EURLEX) author: John Snow Labs name: legclf_united_nations_bert date: 2023-03-06 tags: [en, legal, classification, clauses, united_nations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_united_nations_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class United_Nations or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `United_Nations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_united_nations_bert_en_1.0.0_3.0_1678111655117.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_united_nations_bert_en_1.0.0_3.0_1678111655117.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_united_nations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[United_Nations]| |[Other]| |[Other]| |[United_Nations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_united_nations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.85 0.85 0.85 20 United_Nations 0.87 0.87 0.87 23 accuracy - - 0.86 43 macro-avg 0.86 0.86 0.86 43 weighted-avg 0.86 0.86 0.86 43 ``` --- layout: model title: English BertForQuestionAnswering model (from datauma) author: John Snow Labs name: bert_qa_datauma_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `datauma`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_datauma_bert_finetuned_squad_en_4.0.0_3.0_1654535608413.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_datauma_bert_finetuned_squad_en_4.0.0_3.0_1654535608413.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_datauma_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_datauma_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_datauma").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_datauma_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/datauma/bert-finetuned-squad --- layout: model title: English BertForQuestionAnswering model (from peggyhuang) author: John Snow Labs name: bert_qa_bert_base_uncased_coqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-coqa` is a English model orginally trained by `peggyhuang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_coqa_en_4.0.0_3.0_1654180732707.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_coqa_en_4.0.0_3.0_1654180732707.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_coqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_coqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_coqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/peggyhuang/bert-base-uncased-coqa --- layout: model title: Universal Sentence Encoder Multilingual Large (tfhub_use_multi_lg) author: John Snow Labs name: tfhub_use_multi_lg date: 2021-05-06 tags: [xx, open_source, embeddings] task: Embeddings language: xx edition: Spark NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: UniversalSentenceEncoder article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The Universal Sentence Encoder encodes text into high-dimensional vectors that can be used for text classification, semantic similarity, clustering, and other natural language tasks. The model is trained and optimized for greater-than-word length text, such as sentences, phrases, or short paragraphs. It is trained on a variety of data sources and a variety of tasks with the aim of dynamically accommodating a wide variety of natural language understanding tasks. The input is the variable-length text and the output is a 512-dimensional vector. The universal-sentence-encoder model has trained with a deep averaging network (DAN) encoder. This model supports 16 languages (Arabic, Chinese-simplified, Chinese-traditional, English, French, German, Italian, Japanese, Korean, Dutch, Polish, Portuguese, Spanish, Thai, Turkish, Russian) text encoder. The details are described in the paper "[Multilingual Universal Sentence Encoder for Semantic Retrieval](https://arxiv.org/abs/1907.04307)". Note: This model only works on Linux and macOS operating systems and is not compatible with Windows due to the incompatibility of the SentencePiece library. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/tfhub_use_multi_lg_xx_3.0.0_3.0_1620294638956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/tfhub_use_multi_lg_xx_3.0.0_3.0_1620294638956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_multi_lg", "xx") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") ``` ```scala val embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_multi_lg", "xx") .setInputCols("sentence") .setOutputCol("sentence_embeddings") ``` {:.nlu-block} ```python import nlu text = ["I love NLP", "Me encanta usar SparkNLP"] embeddings_df = nlu.load('xx.use.multi_lg').predict(text, output_level='sentence') embeddings_df ```

## Results ```bash It gives a 512-dimensional vector of the sentences. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|tfhub_use_multi_lg| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|xx| ## Data Source This embeddings model is imported from [https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3](https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3) ## Benchmarking ```bash - We apply this model to the STS benchmark for semantic similarity. Results are shown below: STSBenchmark | dev | test | -----------------------------------|--------|-------| Correlation coefficient of Pearson | 0.837 | 0.825 | - For semantic similarity retrieval, we evaluate the model on [Quora and AskUbuntu retrieval task.](https://arxiv.org/abs/1811.08008). Results are shown below: Dataset | Quora | AskUbuntu | Average | -----------------------|-------|-----------|---------| Mean Average Precision | 89.1 | 42.3 | 65.7 | - For the translation pair retrieval, we evaluate the model on the United Nation Parallel Corpus. Results are shown below: Language Pair | en-es | en-fr | en-ru | en-zh | ---------------|--------|-------|-------|-------| Precision@1 | 86.1 | 83.3 | 88.9 | 78.8 | ``` --- layout: model title: Portuguese asr_bp_sid10_xlsr TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_bp_sid10_xlsr date: 2022-09-26 tags: [wav2vec2, pt, audio, open_source, asr] task: Automatic Speech Recognition language: pt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bp_sid10_xlsr` is a Portuguese model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bp_sid10_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bp_sid10_xlsr_pt_4.2.0_3.0_1664191635179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bp_sid10_xlsr_pt_4.2.0_3.0_1664191635179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bp_sid10_xlsr", "pt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bp_sid10_xlsr", "pt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bp_sid10_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|pt| |Size:|756.4 MB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from janeel) author: John Snow Labs name: roberta_qa_janeel_base_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `janeel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_janeel_base_finetuned_squad_en_4.3.0_3.0_1674217296605.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_janeel_base_finetuned_squad_en_4.3.0_3.0_1674217296605.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_janeel_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_janeel_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_janeel_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/janeel/roberta-base-finetuned-squad --- layout: model title: Pipeline to Detect Clinical Entities author: John Snow Labs name: ner_jsl_greedy_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_greedy](https://nlp.johnsnowlabs.com/2021/06/24/ner_jsl_greedy_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_pipeline_en_3.4.1_3.0_1647869775586.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_pipeline_en_3.4.1_3.0_1647869775586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_jsl_greedy_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("ner_jsl_greedy_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_greedy.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

{% include programmingLanguageSelectScalaPython.html %} ```python model = NerCrfModel.pretrained("ner_crf","en","clinical/models")\ .setInputCols("sentence","token","pos","word_embeddings")\ .setOutputCol("ner") ``` ```scala val model = NerCrfModel.pretrained("ner_crf","en","clinical/models") .setInputCols("sentence","token","pos","word_embeddings") .setOutputCol("ner") ```

{:.model-param} ## Model Information {:.table-model} |---------------|---------------------------------------| | Name: | ner_crf | | Type: | NerCrfModel | | Compatibility: | Spark NLP 2.4.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [sentence, token, pos, word_embeddings] | |Output labels: | [ner] | | Language: | en | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on i2b2 augmented data with `clinical_embeddings` FILLUP --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_hier_triplet_0.1_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223509314.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223509314.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_hier_triplet_0.1_epochs_1_shard_1_squad2.0 --- layout: model title: Extract Clinical Department Entities from Voice of the Patient Documents (embeddings_clinical) author: John Snow Labs name: ner_vop_clinical_dept date: 2023-06-06 tags: [licensed, clinical, en, ner, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts medical devices and clinical department mentions terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `AdmissionDischarge`, `ClinicalDept`, `MedicalDevice` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_en_4.4.3_3.0_1686074506621.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_en_4.4.3_3.0_1686074506621.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------------------|:--------------| | orthopedic department | ClinicalDept | | titanium plate | MedicalDevice | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_clinical_dept| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 AdmissionDischarge 29 1 5 34 0.97 0.85 0.91 ClinicalDept 289 31 37 326 0.90 0.89 0.89 MedicalDevice 253 97 79 332 0.72 0.76 0.74 macro_avg 571 129 121 692 0.86 0.83 0.85 micro_avg 571 129 121 692 0.82 0.83 0.82 ``` --- layout: model title: Arabic Electra Embeddings (from aubmindlab) author: John Snow Labs name: electra_embeddings_araelectra_base_generator date: 2022-05-17 tags: [ar, open_source, electra, embeddings] task: Embeddings language: ar edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `araelectra-base-generator` is a Arabic model orginally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_araelectra_base_generator_ar_3.4.4_3.0_1652786188141.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_araelectra_base_generator_ar_3.4.4_3.0_1652786188141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_araelectra_base_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ar| |Size:|222.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/araelectra-base-generator - https://arxiv.org/pdf/1406.2661.pdf - https://arxiv.org/abs/2012.15516 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: English DistilBertForTokenClassification Cased model (from Neurona) author: John Snow Labs name: distilbert_token_classifier_cpener_test date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cpener-test` is a English model originally trained by `Neurona`. ## Predicted Entities `cpe_vendor`, `cpe_version`, `cpe_product` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_cpener_test_en_4.3.1_3.0_1678783066417.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_cpener_test_en_4.3.1_3.0_1678783066417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_cpener_test| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Neurona/cpener-test --- layout: model title: Legal Warranties Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_warranties_bert date: 2023-03-05 tags: [en, legal, classification, clauses, warranties, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Warranties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Warranties`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_warranties_bert_en_1.0.0_3.0_1678049972315.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_warranties_bert_en_1.0.0_3.0_1678049972315.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_warranties_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Warranties]| |[Other]| |[Other]| |[Warranties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_warranties_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.86 0.87 0.86 83 Warranties 0.81 0.80 0.80 59 accuracy - - 0.84 142 macro-avg 0.83 0.83 0.83 142 weighted-avg 0.84 0.84 0.84 142 ``` --- layout: model title: Modern Greek (1453-) asr_xlsr_53_wav2vec_greek TFWav2Vec2ForCTC from harshit345 author: John Snow Labs name: asr_xlsr_53_wav2vec_greek date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_53_wav2vec_greek` is a Modern Greek (1453-) model originally trained by harshit345. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xlsr_53_wav2vec_greek_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xlsr_53_wav2vec_greek_el_4.2.0_3.0_1664109101429.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xlsr_53_wav2vec_greek_el_4.2.0_3.0_1664109101429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xlsr_53_wav2vec_greek", "el")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xlsr_53_wav2vec_greek", "el") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xlsr_53_wav2vec_greek| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|el| |Size:|1.2 GB| --- layout: model title: Google T5 (Text-To-Text Transfer Transformer) Base author: John Snow Labs name: t5_base date: 2021-01-08 task: [Question Answering, Summarization, Translation] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, t5, summarization, translation, en, seq2seq] supported: true recommended: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The T5 transformer model described in the seminal paper "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer". This model can perform a variety of tasks, such as text summarization, question answering, and translation. More details about using the model can be found in the paper (https://arxiv.org/pdf/1910.10683.pdf). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5TRANSFORMER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_en_2.7.1_2.4_1610133506835.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_en_2.7.1_2.4_1610133506835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Either set the following tasks or have them inline with your input: - summarize: - translate English to German: - translate English to French: - stsb sentence1: Big news. sentence2: No idea. The full list of tasks is in the Appendix of the paper: https://arxiv.org/pdf/1910.10683.pdf

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[t5]| |Language:|en| ## Data Source https://huggingface.co/t5-base --- layout: model title: Voice of the Patients author: John Snow Labs name: ner_vop_wip date: 2023-05-19 tags: [licensed, clinical, en, ner, vop, patient] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `TestResult`, `SubstanceQuantity`, `InjuryOrPoisoning`, `Treatment`, `Modifier`, `HealthStatus`, `MedicalDevice`, `Procedure`, `Symptom`, `Frequency`, `RelationshipStatus`, `Duration`, `Allergen`, `VitalTest`, `Disease`, `Dosage`, `AdmissionDischarge`, `Test`, `Laterality`, `Route`, `DateTime`, `Drug`, `ClinicalDept`, `Vaccine`, `Form`, `Substance`, `PsychologicalCondition`, `Age`, `BodyPart`, `Employment`, `Gender` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_wip_en_4.4.2_3.0_1684508941946.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_wip_en_4.4.2_3.0_1684508941946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token")\ .setContextChars(["*", "-", "“", "(", "[", "\n", ".","\"", "”", ",", "?", ")", "]", "!", ";", ":", "'s", "’s"]) spellModel = ContextSpellCheckerModel\ .pretrained('spellcheck_clinical', 'en', 'clinical/models')\ .setInputCols("token")\ .setOutputCol("checked") pipeline = Pipeline(stages = [ documentAssembler, tokenizer, spellModel]) light_pipeline = LightPipeline(pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) example = ["Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.", "With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.", "Abdomen is sort, nontender, and nonintended.", "Patient not showing pain or any wealth problems.", "No cute distress"] result = light_pipeline.annotate(example) ``` ```scala val assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") .setContextChars(Array("*", "-", "“", "(", "[", "\n", ".","\"", "”", ",", "?", ")", "]", "!", ";", ":", "'s", "’s")) val spellChecker = ContextSpellCheckerModel. pretrained("spellcheck_clinical", "en", "clinical/models"). setInputCols("token"). setOutputCol("checked") val pipeline = new Pipeline().setStages(Array( assembler, tokenizer, spellChecker)) val light_pipeline = new LightPipeline(pipeline.fit(Seq("").toDF("text"))) val text = Array("Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.", "With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.", "Abdomen is sort, nontender, and nonintended.", "Patient not showing pain or any wealth problems.", "No cute distress") val result = light_pipeline.annotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.spell.clinical").predict(""") pipeline = Pipeline(stages = [ documentAssembler, tokenizer, spellModel]) light_pipeline = LightPipeline(pipeline.fit(spark.createDataFrame([[""") ```

## Results ```bash [{'checked': ['With','the','cell','of','physical','therapy','the','patient','was','ambulated','and','on','postoperative',',','the','patient','tolerating','a','post','surgical','soft','diet','.'], 'document': ['Witth the hell of phisical terapy the patient was imbulated and on postoperative, the impatient tolerating a post curgical soft diet.'], 'token': ['Witth','the','hell','of','phisical','terapy','the','patient','was','imbulated','and','on','postoperative',',','the','impatient','tolerating','a','post','curgical','soft','diet','.']}, {'checked': ['With','pain','well','controlled','on','oral','pain','medications',',','she','was','discharged','to','rehabilitation','facility','.'], 'document': ['With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.'], 'token': ['With','paint','wel','controlled','on','orall','pain','medications',',','she','was','discharged','too','reihabilitation','facilitay','.']}, {'checked': ['Abdomen','is','soft',',','nontender',',','and','nondistended','.'], 'document': ['Abdomen is sort, nontender, and nonintended.'], 'token': ['Abdomen','is','sort',',','nontender',',','and','nonintended','.']}, {'checked': ['Patient','not','showing','pain','or','any','health','problems','.'], 'document': ['Patient not showing pain or any wealth problems.'], 'token': ['Patient','not','showing','pain','or','any','wealth','problems','.']}, {'checked': ['No', 'acute', 'distress'], 'document': ['No cute distress'], 'token': ['No', 'cute', 'distress']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_clinical| |Compatibility:|Healthcare NLP 3.4.1| |License:|Licensed| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[corrected]| |Language:|en| |Size:|141.2 MB| ## References MTSamples, i2b2 clinical notes, and several specific medical corpora. --- layout: model title: English DistilBertForQuestionAnswering model (from kaggleodin) author: John Snow Labs name: distilbert_qa_kaggleodin_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `kaggleodin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_kaggleodin_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725640096.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_kaggleodin_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725640096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kaggleodin_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kaggleodin_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_kaggleodin").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_kaggleodin_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kaggleodin/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77 TFWav2Vec2ForCTC from emeson77 author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77` is a English model originally trained by emeson77. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_en_4.2.0_3.0_1664037259824.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77_en_4.2.0_3.0_1664037259824.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_emeson77| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Base Cased model (from mrm8488) author: John Snow Labs name: t5_base_finetuned_break_data date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-finetuned-break_data` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_break_data_en_4.3.0_3.0_1675108822999.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_break_data_en_4.3.0_3.0_1675108822999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_finetuned_break_data","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_finetuned_break_data","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_finetuned_break_data| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|878.4 MB| ## References - https://huggingface.co/mrm8488/t5-base-finetuned-break_data - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/pdf/1910.10683.pdf - https://i.imgur.com/jVFMMWR.png - https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb - https://twitter.com/psuraj28 - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: English asr_wav2vec2_base_checkpoint_6 TFWav2Vec2ForCTC from jiobiala24 author: John Snow Labs name: asr_wav2vec2_base_checkpoint_6 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_checkpoint_6` is a English model originally trained by jiobiala24. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_checkpoint_6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_checkpoint_6_en_4.2.0_3.0_1664020777200.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_checkpoint_6_en_4.2.0_3.0_1664020777200.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_checkpoint_6", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_checkpoint_6", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_checkpoint_6| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|349.2 MB| --- layout: model title: English asr_wav2vec2_murad_with_some_data TFWav2Vec2ForCTC from MBMMurad author: John Snow Labs name: pipeline_asr_wav2vec2_murad_with_some_data date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_murad_with_some_data` is a English model originally trained by MBMMurad. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_murad_with_some_data_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_murad_with_some_data_en_4.2.0_3.0_1664111468486.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_murad_with_some_data_en_4.2.0_3.0_1664111468486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_murad_with_some_data', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_murad_with_some_data", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_murad_with_some_data| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Baltic Languages author: John Snow Labs name: opus_mt_en_bat date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bat, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bat` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bat_xx_2.7.0_2.4_1609166860296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bat_xx_2.7.0_2.4_1609166860296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bat| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from vuiseng9) author: John Snow Labs name: bert_qa_bert_base_squadv1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-squadv1` is a English model orginally trained by `vuiseng9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_squadv1_en_4.0.0_3.0_1654180625513.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_squadv1_en_4.0.0_3.0_1654180625513.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base.by_vuiseng9").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vuiseng9/bert-base-squadv1 --- layout: model title: Stopwords Remover for Polish language (381 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, pl, open_source] task: Stop Words Removal language: pl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_pl_3.4.1_3.0_1646673188092.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_pl_3.4.1_3.0_1646673188092.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------+ |result | +---------------------+ |[jesteś, lepszy, ode]| +---------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|pl| |Size:|2.4 KB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from chiendvhust) author: John Snow Labs name: roberta_qa_chiendvhust_base_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `chiendvhust`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_chiendvhust_base_finetuned_squad_en_4.3.0_3.0_1674217182008.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_chiendvhust_base_finetuned_squad_en_4.3.0_3.0_1674217182008.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_chiendvhust_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_chiendvhust_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_chiendvhust_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|457.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/chiendvhust/roberta-base-finetuned-squad --- layout: model title: Translate English to Sango Pipeline author: John Snow Labs name: translate_en_sg date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, sg, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `sg` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_sg_xx_2.7.0_2.4_1609686246000.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_sg_xx_2.7.0_2.4_1609686246000.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_sg", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_sg", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.sg').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_sg| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Afrikaans (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, af, open_source] task: Embeddings language: af edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_af_3.4.1_3.0_1647281785039.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_af_3.4.1_3.0_1647281785039.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hogger32_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hogger32_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hogger32_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hogger32/distilbert-base-uncased-finetuned-squad --- layout: model title: RCT Binary Classifier (USE) Pipeline author: John Snow Labs name: rct_binary_classifier_use_pipeline date: 2022-06-06 tags: [licensed, rct, clinical, classifier, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [rct_binary_classifier_use](https://nlp.johnsnowlabs.com/2022/05/27/rct_binary_classifier_use_en_3_0.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_RCT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CLASSIFICATION_RCT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rct_binary_classifier_use_pipeline_en_3.4.2_3.0_1654517524887.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rct_binary_classifier_use_pipeline_en_3.4.2_3.0_1654517524887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rct_binary_classifier_use_pipeline", "en", "clinical/models") result = pipeline.annotate("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rct_binary_classifier_use_pipeline", "en", "clinical/models") val result = pipeline.annotate("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.rct_binary_use.pipeline").predict("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ```

## Results ```bash +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |rct |text | +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |true|Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. | +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rct_binary_classifier_use_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|991.2 MB| ## Included Models - DocumentAssembler - UniversalSentenceEncoder - ClassifierDLModel --- layout: model title: Spanish BertForQuestionAnswering model (from MMG) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad date: 2022-06-02 tags: [es, open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-sqac-finetuned-squad` is a Spanish model orginally trained by `MMG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad_es_4.0.0_3.0_1654180513805.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad_es_4.0.0_3.0_1654180513805.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad_sqac.bert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_sqac_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MMG/bert-base-spanish-wwm-cased-finetuned-sqac-finetuned-squad --- layout: model title: Bulgarian Lemmatizer author: John Snow Labs name: lemma date: 2020-05-05 11:14:00 +0800 task: Lemmatization language: bg edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, bg] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_bg_2.5.0_2.4_1588666297763.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_bg_2.5.0_2.4_1588666297763.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "bg") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "bg") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена."""] lemma_df = nlu.load('bg.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Освен', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=6, end=7, result='че', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=9, end=9, result='съм', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=11, end=14, result='крада', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=16, end=17, result='на', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|bg| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from carolgao66) author: John Snow Labs name: distilbert_qa_carolgao66_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `carolgao66`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_carolgao66_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770348185.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_carolgao66_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770348185.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_carolgao66_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_carolgao66_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_carolgao66_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/carolgao66/distilbert-base-uncased-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from Indo-Iranian Languages to English author: John Snow Labs name: opus_mt_iir_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, iir, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `iir` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_iir_en_xx_2.7.0_2.4_1609163487551.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_iir_en_xx_2.7.0_2.4_1609163487551.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_iir_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Dutch RoBERTa Embeddings author: John Snow Labs name: roberta_embeddings_robbert_v2_dutch_base date: 2022-04-14 tags: [roberta, embeddings, nl, open_source] task: Embeddings language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `robbert-v2-dutch-base` is a Dutch model orginally trained by `pdelobelle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robbert_v2_dutch_base_nl_3.4.2_3.0_1649949003731.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robbert_v2_dutch_base_nl_3.4.2_3.0_1649949003731.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_robbert_v2_dutch_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|nl| |Size:|438.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/pdelobelle/robbert-v2-dutch-base - https://github.com/iPieter/RobBERT - https://scholar.google.com/scholar?oi=bibs&hl=en&cites=7180110604335112086 - https://www.aclweb.org/anthology/2021.wassa-1.27/ - https://arxiv.org/pdf/2001.06286.pdf - https://biblio.ugent.be/publication/8704637/file/8704638.pdf - https://arxiv.org/pdf/2001.06286.pdf - https://arxiv.org/pdf/2001.06286.pdf - https://arxiv.org/pdf/2004.02814.pdf - https://github.com/proycon/deepfrog - https://arxiv.org/pdf/2001.06286.pdf - https://github.com/proycon/deepfrog - https://arxiv.org/pdf/2001.06286.pdf - https://arxiv.org/pdf/2010.13652.pdf - https://www.cambridge.org/core/journals/natural-language-engineering/article/abs/automatic-classification-of-participant-roles-in-cyberbullying-can-we-detect-victims-bullies-and-bystanders-in-social-media-text/A2079C2C738C29428E666810B8903342 - https://gitlab.com/spelfouten/dutch-simpletransformers/ - https://arxiv.org/pdf/2101.05716.pdf - https://medium.com/broadhorizon-cmotions/nlp-with-r-part-5-state-of-the-art-in-nlp-transformers-bert-3449e3cd7494 - https://people.cs.kuleuven.be/~pieter.delobelle/robbert/ - https://arxiv.org/abs/2001.06286 - https://github.com/iPieter/RobBERT - https://arxiv.org/abs/1907.11692 - https://github.com/pytorch/fairseq/tree/master/examples/roberta - https://people.cs.kuleuven.be/~pieter.delobelle/robbert/ - https://arxiv.org/abs/2001.06286 - https://github.com/iPieter/RobBERT - https://github.com/benjaminvdb/110kDBRD - https://www.statmt.org/europarl/ - https://arxiv.org/abs/2001.02943 - https://universaldependencies.org/treebanks/nl_lassysmall/index.html - https://www.clips.uantwerpen.be/conll2002/ner/ - https://oscar-corpus.com/ - https://github.com/pytorch/fairseq/tree/master/examples/roberta - https://github.com/pytorch/fairseq/tree/master/examples/roberta - https://arxiv.org/abs/2001.06286 - https://github.com/iPieter/RobBERT#how-to-replicate-our-paper-experiments - https://arxiv.org/abs/1909.11942 - https://camembert-model.fr/ - https://en.wikipedia.org/wiki/Robbert - https://muppet.fandom.com/wiki/Bert - https://github.com/iPieter/RobBERT/blob/master/res/robbert_logo.png - https://people.cs.kuleuven.be/~pieter.delobelle - https://thomaswinters.be - https://people.cs.kuleuven.be/~bettina.berendt/ --- layout: model title: Word2Vec Embeddings in Basque (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, eu, open_source] task: Embeddings language: eu edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eu_3.4.1_3.0_1647285496461.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eu_3.4.1_3.0_1647285496461.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | document | sentence | token | normal | |---:|:-----------|:-----------|:----------|:----------| | 0 | ['Hello'] | ['Hello'] | ['Hello'] | ['Hello'] || | document | sentence | token | normal | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clean_pattern| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: Detect Assertion Status (assertion_wip) author: John Snow Labs name: jsl_assertion_wip date: 2021-01-18 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.7.0 spark_version: 2.4 tags: [clinical, licensed, assertion, en, ner] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The deep neural network architecture for assertion status detection in Spark NLP is based on a BiLSTM framework, and is a modified version of the architecture proposed by Fancellu et.al. (Fancellu, Lopez, and Webber 2016). Its goal is to classify the assertions made on given medical concepts as being present, absent, or possible in the patient, conditionally present in the patient under certain circumstances, hypothetically present in the patient at some future point, and mentioned in the patient report but associated with someoneelse (Uzuner et al. 2011). {:.h2_title} ## Predicted Entities `Present`, `Absent`, `Possible`, `Planned`, `Someoneelse`, `Past`, `Family`, `None`, `Hypotetical`. {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_assertion_wip_en_2.6.1_2.4_1606860510166.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_assertion_wip_en_2.6.1_2.4_1606860510166.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel, NerConverter, AssertionDLModel.

{% include programmingLanguageSelectScalaPython.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") clinical_assertion = AssertionDLModel.pretrained("jsl_assertion_wip", "en", "clinical/models") \ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = model.transform(spark.createDataFrame([["The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature."]], ["text"]) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = NerDLModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val nerConverter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val clinical_assertion = AssertionDLModel.pretrained("jsl_assertion_wip", "en", "clinical/models") .setInputCols(Array("sentence", "ner_chunk", "embeddings")) .setOutputCol("assertion") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, clinical_assertion)) val data = Seq("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results The output is a dataframe with a sentence per row and an ``"assertion"`` column containing all of the assertion labels in the sentence. The assertion column also contains assertion character indices, and other metadata. To get only the entity chunks and assertion labels, without the metadata, select ``"ner_chunk.result"`` and ``"assertion.result"`` from your output dataframe. ```bash +-----------------------------------------+-----+---+----------------------------+-------+---------+ |chunk |begin|end|ner_label |sent_id|assertion| +-----------------------------------------+-----+---+----------------------------+-------+---------+ |21-day-old |17 |26 |Age |0 |Family | |Caucasian |28 |36 |Race_Ethnicity |0 |Family | |male |38 |41 |Gender |0 |Family | |for 2 days |48 |57 |Duration |0 |Family | |congestion |62 |71 |Symptom |0 |Present | |mom |75 |77 |Gender |0 |Family | |yellow |99 |104|Modifier |0 |Family | |discharge |106 |114|Symptom |0 |Family | |nares |135 |139|External_body_part_or_region|0 |Family | |she |147 |149|Gender |0 |Family | |mild |168 |171|Modifier |0 |Family | |problems with his breathing while feeding|173 |213|Symptom |0 |Present | |perioral cyanosis |237 |253|Symptom |0 |Absent | |retractions |258 |268|Symptom |0 |Absent | |One day ago |272 |282|RelativeDate |1 |Family | |mom |285 |287|Gender |1 |Family | |Tylenol |345 |351|Drug_BrandName |1 |Family | |Baby |354 |357|Age |2 |Family | |decreased p.o. intake |377 |397|Symptom |2 |Family | |His |400 |402|Gender |3 |Family | +-----------------------------------------+-----+---+----------------------------+-------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_assertion_wip| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence, ner_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|[en]| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with 'embeddings_clinical'. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ {:.h2_title} ## Benchmarking ```bash label prec rec f1 Absent 0.970 0.943 0.956 Someoneelse 0.868 0.775 0.819 Planned 0.721 0.754 0.737 Possible 0.852 0.884 0.868 Past 0.811 0.823 0.817 Present 0.833 0.866 0.849 Family 0.872 0.921 0.896 None 0.609 0.359 0.452 Hypothetical 0.722 0.810 0.763 Macro-average 0.888 0.872 0.880 Micro-average 0.908 0.908 0.908 ``` --- layout: model title: English DistilBertForQuestionAnswering Cased model (from saraks) author: John Snow Labs name: distilbert_qa_cuad_parties_cased_08_31_v1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cuad-distil-parties-cased-08-31-v1` is a English model originally trained by `saraks`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_parties_cased_08_31_v1_en_4.3.0_3.0_1672766262518.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_parties_cased_08_31_v1_en_4.3.0_3.0_1672766262518.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_parties_cased_08_31_v1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_parties_cased_08_31_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cuad_parties_cased_08_31_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saraks/cuad-distil-parties-cased-08-31-v1 --- layout: model title: Ganda asr_wav2vec2_xlsr_multilingual_56 TFWav2Vec2ForCTC from voidful author: John Snow Labs name: asr_wav2vec2_xlsr_multilingual_56 date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_multilingual_56` is a Ganda model originally trained by voidful. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_multilingual_56_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_multilingual_56_lg_4.2.0_3.0_1664035818043.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_multilingual_56_lg_4.2.0_3.0_1664035818043.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_multilingual_56", "lg")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_multilingual_56", "lg") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_multilingual_56| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lg| |Size:|1.2 GB| --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_becasincentivos2 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-becasIncentivos2` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos2_es_4.3.0_3.0_1674218030841.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos2_es_4.3.0_3.0_1674218030841.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos2","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_becasincentivos2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|459.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-becasIncentivos2 --- layout: model title: English RobertaForQuestionAnswering (from sunitha) author: John Snow Labs name: roberta_qa_roberta_customds_finetune date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-customds-finetune` is a English model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_customds_finetune_en_4.0.0_3.0_1655735713085.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_customds_finetune_en_4.0.0_3.0_1655735713085.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_customds_finetune","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_customds_finetune","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.by_sunitha").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_customds_finetune| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/roberta-customds-finetune --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from abhinavkulkarni) author: John Snow Labs name: distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `abhinavkulkarni`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769656788.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769656788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_abhinavkulkarni_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/abhinavkulkarni/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate Waray to English Pipeline author: John Snow Labs name: translate_war_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, war, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `war` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_war_en_xx_2.7.0_2.4_1609700727408.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_war_en_xx_2.7.0_2.4_1609700727408.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_war_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_war_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.war.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_war_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal The merger Clause Binary Classifier author: John Snow Labs name: legclf_the_merger_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `the-merger` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `the-merger` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_the_merger_clause_en_1.0.0_3.2_1660123103140.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_the_merger_clause_en_1.0.0_3.2_1660123103140.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_the_merger_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[the-merger]| |[other]| |[other]| |[the-merger]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_the_merger_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.98 1.00 0.99 98 the-merger 1.00 0.95 0.97 38 accuracy - - 0.99 136 macro-avg 0.99 0.97 0.98 136 weighted-avg 0.99 0.99 0.99 136 ``` --- layout: model title: Pipeline to Detect Clinical Events author: John Snow Labs name: ner_events_healthcare_pipeline date: 2022-03-22 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_events_healthcare](https://nlp.johnsnowlabs.com/2021/04/01/ner_events_healthcare_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_EVENTS_CLINICAL.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_pipeline_en_3.4.1_3.0_1647943997404.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_pipeline_en_3.4.1_3.0_1647943997404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_events_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("The patient presented to the emergency room last evening") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_events_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("The patient presented to the emergency room last evening") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.healthcare_events.pipeline").predict("""The patient presented to the emergency room last evening""") ```

## Results ```bash +------------------+-------------+ |chunks |entities | +------------------+-------------+ |presented |EVIDENTIAL | |the emergency room|CLINICAL_DEPT| |last evening |DATE | +------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Translate English to Chuukese Pipeline author: John Snow Labs name: translate_en_chk date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, chk, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `chk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_chk_xx_2.7.0_2.4_1609689317048.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_chk_xx_2.7.0_2.4_1609689317048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_chk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_chk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.chk').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_chk| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from dmis-lab) author: John Snow Labs name: bert_qa_biobert_base_cased_v1.1_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobert-base-cased-v1.1-squad` is a English model orginally trained by `dmis-lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_en_4.0.0_3.0_1654185575469.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biobert_base_cased_v1.1_squad_en_4.0.0_3.0_1654185575469.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biobert_base_cased_v1.1_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biobert_base_cased_v1.1_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.biobert.base_cased.by_dmis-lab").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biobert_base_cased_v1.1_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/dmis-lab/biobert-base-cased-v1.1-squad --- layout: model title: Korean Lemmatizer author: John Snow Labs name: lemma date: 2021-01-15 task: Lemmatization language: ko edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ko, lemmatizer, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_ko_2.7.0_2.4_1610747055280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_ko_2.7.0_2.4_1610747055280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.pretrained('wordseg_kaist_ud', 'ko')\ .setInputCols("document")\ .setOutputCol("token") lemmatizer = LemmatizerModel.pretrained("lemma", "ko") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, word_segmenter , lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) results = light_pipeline.fullAnnotate(["이렇게되면이러한인간형을다투어본받으려할것이틀림없다."]) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_kaist_ud", "ko") .setInputCols("document") .setOutputCol("token") val lemmatizer = LemmatizerModel.pretrained("lemma", "ko") .setInputCols("token") .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter , lemmatizer)) val data = Seq("이렇게되면이러한인간형을다투어본받으려할것이틀림없다.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["이렇게되면이러한인간형을다투어본받으려할것이틀림없다."] lemma_df = nlu.load('ko.lemma').predict(text, output_level = "document") lemma_df.lemma.values[0] ```

## Results ```bash {'lemma': [Annotation(token, 0, 2, 이렇게, {'sentence': '0'}), Annotation(token, 3, 4, 되+면, {'sentence': '0'}), Annotation(token, 5, 7, 이러한+ㄴ, {'sentence': '0'}), Annotation(token, 8, 11, 인간형+을, {'sentence': '0'}), Annotation(token, 12, 15, 다투어본, {'sentence': '0'}), Annotation(token, 16, 18, 받으할, {'sentence': '0'}), Annotation(token, 18, 18, 려, {'sentence': '0'}), Annotation(token, 20, 21, 것+이, {'sentence': '0'}), Annotation(token, 22, 25, 틀림없+다, {'sentence': '0'}), Annotation(token, 26, 26, ., {'sentence': '0'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|ko| ## Data Source The model was trained on the universal dependencies from _Korea Advanced Institute of Science and Technology (KAIST)_ dataset. Reference: - Building Universal Dependency Treebanks in Korean, Jayeol Chun, Na-Rae Han, Jena D. Hwang, and Jinho D. Choi. In Proceedings of the 11th International Conference on Language Resources and Evaluation, LREC'18, Miyazaki, Japan, 2018. --- layout: model title: Legal Choice of law Clause Binary Classifier (md) author: John Snow Labs name: legclf_choice_of_law_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `choice-of-law` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `choice-of-law` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_choice_of_law_md_en_1.0.0_3.0_1673460245223.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_choice_of_law_md_en_1.0.0_3.0_1673460245223.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_choice_of_law_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[choice-of-law]| |[other]| |[other]| |[choice-of-law]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_choice_of_law_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents + Lawinsider categorization ## Benchmarking ```bash precision recall f1-score support amendments-and-waivers 1.00 0.97 0.99 35 other 0.97 1.00 0.99 39 accuracy 0.99 74 macro avg 0.99 0.99 0.99 74 weighted avg 0.99 0.99 0.99 74 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from ThomasNLG) author: John Snow Labs name: t5_qa_webnlg_synth date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-qa_webnlg_synth-en` is a English model originally trained by `ThomasNLG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_qa_webnlg_synth_en_4.3.0_3.0_1675125486836.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_qa_webnlg_synth_en_4.3.0_3.0_1675125486836.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_qa_webnlg_synth","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_qa_webnlg_synth","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_qa_webnlg_synth| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|271.7 MB| ## References - https://huggingface.co/ThomasNLG/t5-qa_webnlg_synth-en - https://github.com/ThomasScialom/QuestEval - https://arxiv.org/abs/2104.07555 --- layout: model title: Legal Position and duties Clause Binary Classifier author: John Snow Labs name: legclf_position_and_duties_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `position-and-duties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `position-and-duties` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_position_and_duties_clause_en_1.0.0_3.2_1660122849201.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_position_and_duties_clause_en_1.0.0_3.2_1660122849201.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_position_and_duties_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[position-and-duties]| |[other]| |[other]| |[position-and-duties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_position_and_duties_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.99 0.99 92 position-and-duties 0.97 1.00 0.99 38 accuracy - - 0.99 130 macro-avg 0.99 0.99 0.99 130 weighted-avg 0.99 0.99 0.99 130 ``` --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-igbo-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `PER`, `DATE`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili_sw_4.1.0_3.0_1659354042745.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili_sw_4.1.0_3.0_1659354042745.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_igbo_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-igbo-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Translate English to Mossi Pipeline author: John Snow Labs name: translate_en_mos date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, mos, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `mos` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_mos_xx_2.7.0_2.4_1609686900824.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_mos_xx_2.7.0_2.4_1609686900824.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_mos", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_mos", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.mos').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_mos| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_25_1Aug_2022 TFWav2Vec2ForCTC from Roshana author: John Snow Labs name: pipeline_asr_wav2vec2_25_1Aug_2022 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_25_1Aug_2022` is a English model originally trained by Roshana. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_25_1Aug_2022_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_25_1Aug_2022_en_4.2.0_3.0_1664116524444.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_25_1Aug_2022_en_4.2.0_3.0_1664116524444.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_25_1Aug_2022', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_25_1Aug_2022", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_25_1Aug_2022| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dl4 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dl4` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl4_en_4.3.0_3.0_1675110033833.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl4_en_4.3.0_3.0_1675110033833.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dl4","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dl4","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dl4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|329.9 MB| ## References - https://huggingface.co/google/t5-efficient-base-dl4 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Translate English to Basque (family) Pipeline author: John Snow Labs name: translate_en_euq date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, euq, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `euq` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_euq_xx_2.7.0_2.4_1609689947319.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_euq_xx_2.7.0_2.4_1609689947319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_euq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_euq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.euq').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_euq| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from franklu) author: John Snow Labs name: bert_qa_pubmed_bert_squadv2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `pubmed_bert_squadv2` is a English model orginally trained by `franklu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_pubmed_bert_squadv2_en_4.0.0_3.0_1654189059722.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_pubmed_bert_squadv2_en_4.0.0_3.0_1654189059722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_pubmed_bert_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_pubmed_bert_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_pubmed.bert.v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_pubmed_bert_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/franklu/pubmed_bert_squadv2 - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578 TFWav2Vec2ForCTC from doddle124578 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578` is a English model originally trained by doddle124578. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_en_4.2.0_3.0_1664037306955.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578_en_4.2.0_3.0_1664037306955.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_1_by_doddle124578| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Oncology Pipeline for Biomarkers author: John Snow Labs name: oncology_biomarker_pipeline date: 2023-03-29 tags: [licensed, pipeline, oncology, biomarker, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition, Assertion Status and Relation Extraction models to extract information from oncology texts. This pipeline focuses on entities related to biomarkers. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.3.2_3.2_1680112789514.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.3.2_3.2_1680112789514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") text = '''Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") val text = "Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_biomarker.pipeline").predict("""Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.""") ```

## Results ```bash " ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_oncology_biomarker_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Biomarker | ******************** ner_oncology_test_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | Immunohistochemistry | Pathology_Test | | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_biomarker results ******************** | chunk | ner_label | |:-------------------------------|:----------------------| | Immunohistochemistry | Test | | negative | Biomarker_Measurement | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Measurement | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Measurement | | HER2 | Biomarker | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:------------| | Immunohistochemistry | Pathology_Test | Past | | thyroid transcription factor-1 | Biomarker | Present | | napsin A | Biomarker | Present | | ER | Biomarker | Present | | PR | Biomarker | Present | | HER2 | Oncogene | Present | ******************** assertion_oncology_test_binary_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:----------------| | Immunohistochemistry | Pathology_Test | Medical_History | | thyroid transcription factor-1 | Biomarker | Medical_History | | napsin A | Biomarker | Medical_History | | ER | Biomarker | Medical_History | | PR | Biomarker | Medical_History | | HER2 | Oncogene | Medical_History | ******************** re_oncology_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_related_to | | negative | Biomarker_Result | napsin A | Biomarker | is_related_to | | positive | Biomarker_Result | ER | Biomarker | is_related_to | | positive | Biomarker_Result | PR | Biomarker | is_related_to | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_related_to | ******************** re_oncology_granular_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | is_finding_of | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ******************** re_oncology_biomarker_result_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | is_finding_of | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_biomarker_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - RelationExtractionModel - RelationExtractionModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from NitishKumar) author: John Snow Labs name: distilbert_qa_nitishkumar_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `NitishKumar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_nitishkumar_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768854083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_nitishkumar_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768854083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_nitishkumar_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_nitishkumar_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_nitishkumar_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/NitishKumar/distilbert-base-uncased-finetuned-squad --- layout: model title: Urdu Bert Embeddings (from Geotrend) author: John Snow Labs name: bert_embeddings_bert_base_ur_cased date: 2022-04-11 tags: [bert, embeddings, ur, open_source] task: Embeddings language: ur edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-ur-cased` is a Urdu model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_ur_cased_ur_3.4.2_3.0_1649676499960.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_ur_cased_ur_3.4.2_3.0_1649676499960.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_ur_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ur| |Size:|348.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-ur-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Multilingual XLMRobertaForTokenClassification Cased model (from magistermilitum) author: John Snow Labs name: xlmroberta_ner_roberta_multilingual_medieval date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-multilingual-medieval-ner` is a Multilingual model originally trained by `magistermilitum`. ## Predicted Entities `LOC`, `L-PERS`, `PERS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_roberta_multilingual_medieval_xx_4.1.0_3.0_1660422872636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_roberta_multilingual_medieval_xx_4.1.0_3.0_1660422872636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_roberta_multilingual_medieval","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_roberta_multilingual_medieval","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_roberta_multilingual_medieval| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/magistermilitum/roberta-multilingual-medieval-ner --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_1_lr_2e_5_bs_32_ep_3 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-set_date_1-lr-2e-5-bs-32-ep-3` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_lr_2e_5_bs_32_ep_3_en_4.0.0_3.0_1657188305649.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_lr_2e_5_bs_32_ep_3_en_4.0.0_3.0_1657188305649.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_lr_2e_5_bs_32_ep_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_lr_2e_5_bs_32_ep_3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_1_lr_2e_5_bs_32_ep_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-set_date_1-lr-2e-5-bs-32-ep-3 --- layout: model title: Arabic ElectraForQuestionAnswering model (from aymanm419) Version-1 author: John Snow Labs name: electra_qa_araElectra_SQUAD_ARCD date: 2022-06-22 tags: [ar, open_source, electra, question_answering] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `araElectra-SQUAD-ARCD` is a Arabic model originally trained by `aymanm419`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_araElectra_SQUAD_ARCD_ar_4.0.0_3.0_1655920164320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_araElectra_SQUAD_ARCD_ar_4.0.0_3.0_1655920164320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_araElectra_SQUAD_ARCD","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_araElectra_SQUAD_ARCD","ar") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.squad_arcd.electra").predict("""ما هو اسمي؟|||"اسمي كلارا وأنا أعيش في بيركلي.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_araElectra_SQUAD_ARCD| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ar| |Size:|504.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aymanm419/araElectra-SQUAD-ARCD --- layout: model title: Translate English to Multiple languages Pipeline author: John Snow Labs name: translate_en_mul date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, mul, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `mul` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_mul_xx_2.7.0_2.4_1609689926198.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_mul_xx_2.7.0_2.4_1609689926198.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_mul", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_mul", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.mul').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_mul| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from English to Indo-European Languages author: John Snow Labs name: opus_mt_en_ine date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ine, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ine` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ine_xx_2.7.0_2.4_1609164123331.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ine_xx_2.7.0_2.4_1609164123331.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ine| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: RoBERTa Large CoNLL-03 NER Pipeline author: ahmedlone127 name: roberta_large_token_classifier_conll03_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, roberta, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/roberta_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/roberta_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655220223619.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/roberta_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655220223619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("roberta_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("roberta_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.")) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_motiondew_finetuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_motiondew_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_motiondew_finetuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/distilbert-finetuned --- layout: model title: Legal Question Answering (Bert, Large) author: John Snow Labs name: legqa_bert_large date: 2022-08-09 tags: [en, legal, qa, licensed] task: Question Answering language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Legal Bert-based Question Answering model, trained on squad-v2, finetuned on proprietary Legal questions and answers. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legqa_bert_large_en_1.0.0_3.2_1660053509660.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legqa_bert_large_en_1.0.0_3.2_1660053509660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash `her husband` ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legqa_bert_large| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References Trained on squad-v2, finetuned on proprietary Legal questions and answers. --- layout: model title: RE Pipeline between Body Parts and Procedures author: John Snow Labs name: re_bodypart_proceduretest_pipeline date: 2022-03-31 tags: [licensed, clinical, relation_extraction, body_part, procedures, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_bodypart_proceduretest](https://nlp.johnsnowlabs.com/2021/01/18/re_bodypart_proceduretest_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_BODYPART_ENT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_bodypart_proceduretest_pipeline_en_3.4.1_3.0_1648733647318.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_bodypart_proceduretest_pipeline_en_3.4.1_3.0_1648733647318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_bodypart_proceduretest_pipeline", "en", "clinical/models") pipeline.fullAnnotate("TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_bodypart_proceduretest_pipeline", "en", "clinical/models") pipeline.fullAnnotate("TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.bodypart_proceduretest.pipeline").predict("""TECHNIQUE IN DETAIL: After informed consent was obtained from the patient and his mother, the chest was scanned with portable ultrasound.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "ro", "clinical/models") sample = """Medic : Dr. Agota EVELYN, C.N.P : 2450502264401, Data setului de analize: 25 May 2022 Varsta : 77, Nume si Prenume : BUREAN MARIA Tel: +40(235)413773, E-mail : hale@gmail.com, Licență : B004256985M, Înmatriculare : CD205113, Cont : FXHZ7170951927104999, Spitalul Pentru Ochi de Deal Drumul Oprea Nr. 972 Vaslui, 737405 """ result = deid_pipeline.annotate(sample) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification", "ro", "clinical/models") val sample = """Medic : Dr. Agota EVELYN, C.N.P : 2450502264401, Data setului de analize: 25 May 2022 Varsta : 77, Nume si Prenume : BUREAN MARIA Tel: +40(235)413773, E-mail : hale@gmail.com, Licență : B004256985M, Înmatriculare : CD205113, Cont : FXHZ7170951927104999, Spitalul Pentru Ochi de Deal Drumul Oprea Nr. 972 Vaslui, 737405 """ val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("ro.deid.clinical").predict("""Medic : Dr. Agota EVELYN, C.N.P : 2450502264401, Data setului de analize: 25 May 2022 Varsta : 77, Nume si Prenume : BUREAN MARIA Tel: +40(235)413773, E-mail : hale@gmail.com, Licență : B004256985M, Înmatriculare : CD205113, Cont : FXHZ7170951927104999, Spitalul Pentru Ochi de Deal Drumul Oprea Nr. 972 Vaslui, 737405 """) ```

## Results ```bash Results Masked with entity labels ------------------------------ Medic : Dr. , C.N.P : , Data setului de analize: Varsta : , Nume si Prenume : Tel: , E-mail : , Licență : , Înmatriculare : , Cont : , , Masked with chars ------------------------------ Medic : Dr. [**********], C.N.P : [***********], Data setului de analize: [*********] Varsta : **, Nume si Prenume : [**********] Tel: [************], E-mail : [************], Licență : [*********], Înmatriculare : [******], Cont : [******************], [**************************] [******************] [****], [****] Masked with fixed length chars ------------------------------ Medic : Dr. ****, C.N.P : ****, Data setului de analize: **** Varsta : ****, Nume si Prenume : **** Tel: ****, E-mail : ****, Licență : ****, Înmatriculare : ****, Cont : ****, **** **** ****, **** Obfuscated ------------------------------ Medic : Dr. Doina Gheorghiu, C.N.P : 6794561192919, Data setului de analize: 01-04-2001 Varsta : 91, Nume si Prenume : Dragomir Emilia Tel: 0248 551 376, E-mail : tudorsmaranda@kappa.ro, Licență : T003485962M, Înmatriculare : AR-65-UPQ, Cont : KHHO5029180812813651, Centrul Medical de Evaluare si Recuperare pentru Copii si Tineri Cristian Serban Buzias Aleea Voinea Curcani, 328479 {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|ro| |Size:|1.2 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English asr_wav2vec2_large_960h_lv60 TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_large_960h_lv60 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h_lv60` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_960h_lv60_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_en_4.2.0_3.0_1664017406218.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_en_4.2.0_3.0_1664017406218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_960h_lv60', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_960h_lv60", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_960h_lv60| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_10_h_768 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-10_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_768_zh_4.2.4_3.0_1670021510353.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_768_zh_4.2.4_3.0_1670021510353.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_10_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|330.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-10_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Legal Amendments Clause Binary Classifier author: John Snow Labs name: legclf_amendments_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `amendments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `amendments` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_amendments_clause_en_1.0.0_3.2_1660122105448.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_amendments_clause_en_1.0.0_3.2_1660122105448.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_amendments_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[amendments]| |[other]| |[other]| |[amendments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_amendments_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support amendments 0.95 0.96 0.95 227 other 0.99 0.98 0.98 673 accuracy - - 0.98 900 macro-avg 0.97 0.97 0.97 900 weighted-avg 0.98 0.98 0.98 900 ``` --- layout: model title: Legal Intercreditor Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_intercreditor_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, intercreditor, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_intercreditor_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `intercreditor-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `intercreditor-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_intercreditor_agreement_bert_en_1.0.0_3.0_1669314986657.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_intercreditor_agreement_bert_en_1.0.0_3.0_1669314986657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_intercreditor_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[intercreditor-agreement]| |[other]| |[other]| |[intercreditor-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_intercreditor_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support intercreditor-agreement 0.87 0.82 0.84 33 other 0.93 0.95 0.94 82 accuracy - - 0.91 115 macro-avg 0.90 0.88 0.89 115 weighted-avg 0.91 0.91 0.91 115 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from ModelTC) author: John Snow Labs name: roberta_qa_modeltc_base_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad` is a English model originally trained by `ModelTC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_modeltc_base_squad_en_4.3.0_3.0_1674218615238.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_modeltc_base_squad_en_4.3.0_3.0_1674218615238.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_modeltc_base_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_modeltc_base_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_modeltc_base_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ModelTC/roberta-base-squad --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4_en_4.3.0_3.0_1674214918693.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4_en_4.3.0_3.0_1674214918693.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_256_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|427.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-4 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-42` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42_en_4.3.0_3.0_1674214395082.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42_en_4.3.0_3.0_1674214395082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_16_finetuned_squad_seed_42| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|425.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-42 --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-42` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42_en_4.0.0_3.0_1655731309098.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42_en_4.0.0_3.0_1655731309098.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_128d_seed_42").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_42| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|430.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-42 --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_de_4.2.0_3.0_1664117864895.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_de_4.2.0_3.0_1664117864895.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Portuguese Part of Speech Tagger (from Emanuel) author: John Snow Labs name: bert_pos_autonlp_pos_tag_bosque date: 2022-05-09 tags: [bert, pos, part_of_speech, pt, open_source] task: Part of Speech Tagging language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `autonlp-pos-tag-bosque` is a Portuguese model orginally trained by `Emanuel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_autonlp_pos_tag_bosque_pt_3.4.2_3.0_1652091764630.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_autonlp_pos_tag_bosque_pt_3.4.2_3.0_1652091764630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_autonlp_pos_tag_bosque| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|pt| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Emanuel/autonlp-pos-tag-bosque --- layout: model title: Portuguese Lemmatizer author: John Snow Labs name: lemma date: 2020-05-03 12:54:00 +0800 task: Lemmatization language: pt edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, pt] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_pt_2.5.0_2.4_1588499301752.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_pt_2.5.0_2.4_1588499301752.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "pt") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "pt") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Além de ser o rei do norte, John Snow é um médico inglês e líder no desenvolvimento de anestesia e higiene médica."""] lemma_df = nlu.load('pt.lemma').predict(text, output_level='token') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=3, result='Além', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=5, end=6, result='de', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=8, end=10, result='ser', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=12, end=12, result='o', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=14, end=16, result='rei', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|pt| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Context Spell Checker Pipeline for English author: John Snow Labs name: spellcheck_dl_pipeline date: 2022-04-18 tags: [spellcheck, spell, spellcheck_pipeline, spelling_corrector, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained spellchecker pipeline is built on the top of [spellcheck_dl](https://nlp.johnsnowlabs.com/2022/04/02/spellcheck_dl_en_2_4.html) model. This pipeline is for PySpark 2.4.x users with SparkNLP 3.4.2 and above. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.2_2.4_1650285592232.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.2_2.4_1650285592232.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("spellcheck_dl_pipeline", lang = "en") text = ["During the summer we have the best ueather.", "I have a black ueather jacket, so nice."] pipeline.annotate(text) ``` ```scala val pipeline = new PretrainedPipeline("spellcheck_dl_pipeline", lang = "en") val example = Array("During the summer we have the best ueather.", "I have a black ueather jacket, so nice.") pipeline.annotate(example) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_fi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_fi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.fi.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_fi_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Russian T5ForConditionalGeneration Small Cased model (from cointegrated) author: John Snow Labs name: t5_rut5_small_chitchat date: 2023-01-30 tags: [ru, open_source, t5, tensorflow] task: Text Generation language: ru edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rut5-small-chitchat` is a Russian model originally trained by `cointegrated`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_rut5_small_chitchat_ru_4.3.0_3.0_1675106805725.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_rut5_small_chitchat_ru_4.3.0_3.0_1675106805725.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_rut5_small_chitchat","ru") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_rut5_small_chitchat","ru") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_rut5_small_chitchat| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ru| |Size:|277.4 MB| ## References - https://huggingface.co/cointegrated/rut5-small-chitchat --- layout: model title: Legal Further Assurances Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_further_assurances_bert date: 2023-03-05 tags: [en, legal, classification, clauses, further_assurances, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Further_Assurances` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Further_Assurances`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_further_assurances_bert_en_1.0.0_3.0_1678050719026.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_further_assurances_bert_en_1.0.0_3.0_1678050719026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_further_assurances_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Further_Assurances]| |[Other]| |[Other]| |[Further_Assurances]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_further_assurances_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Further_Assurances 0.93 0.97 0.95 120 Other 0.98 0.94 0.96 147 accuracy - - 0.96 267 macro-avg 0.95 0.96 0.95 267 weighted-avg 0.96 0.96 0.96 267 ``` --- layout: model title: Classifier for Adverse Drug Events in Small Conversations author: John Snow Labs name: classifierdl_ade_conversational_biobert date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.7.1 spark_version: 2.4 tags: [en, licensed, classifier, clinical] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify sentence in two categories: `True` : The sentence is talking about a possible ADE `False` : The sentences doesn’t have any information about an ADE. ## Predicted Entities `True`, `False` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_conversational_biobert_en_2.7.1_2.4_1611246389884.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_conversational_biobert_en_2.7.1_2.4_1611246389884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler().setInputCol("text").setOutputCol("document") tokenizer = Tokenizer().setInputCols(['document']).setOutputCol('token') embeddings = BertEmbeddings.pretrained('biobert_pubmed_base_cased')\ .setInputCols(["document", 'token'])\ .setOutputCol("word_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["document", "word_embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifier = ClassifierDLModel.pretrained('classifierdl_ade_conversational_biobert', 'en', 'clinical/models')\ .setInputCols(['document', 'token', 'sentence_embeddings']).setOutputCol('class') nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, embeddings, sentence_embeddings, classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate(["I feel a bit drowsy & have a little blurred vision after taking an insulin", "I feel great after taking tylenol"]) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.ade.conversational").predict("""I feel a bit drowsy & have a little blurred vision after taking an insulin""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_768", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_768", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L10_768').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L10_768_embeddings sentence [-0.6537564396858215, -0.2422734946012497, -0.... I hate cancer [0.06436929106712341, -0.34515661001205444, 0.... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L10_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1 --- layout: model title: Pipeline to Extract Negation and Uncertainty Entities from Spanish Medical Texts author: John Snow Labs name: ner_negation_uncertainty_pipeline date: 2023-03-09 tags: [es, clinical, licensed, ner, unc, usco, neg, nsco, negation, uncertainty] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_negation_uncertainty](https://nlp.johnsnowlabs.com/2022/08/13/ner_negation_uncertainty_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_negation_uncertainty_pipeline_es_4.3.0_3.2_1678359171669.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_negation_uncertainty_pipeline_es_4.3.0_3.2_1678359171669.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_negation_uncertainty_pipeline", "es", "clinical/models") text = '''e realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_negation_uncertainty_pipeline", "es", "clinical/models") val text = "e realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa)." val result = pipeline.fullAnnotate(text) ```

## Results ```bash +------------------------------------------------------+---------+ |chunk |ner_label| +------------------------------------------------------+---------+ |probable de |UNC | |cirrosis hepática |USCO | |no |NEG | |conocida previamente |NSCO | |no |NEG | |se realizó paracentesis control por escasez de liquido|NSCO | |susceptible de |UNC | |ca basocelular perlado |USCO | +------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_negation_uncertainty_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|318.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Sentence Detection in Bosnian Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [bs, sentence_detection, open_source] task: Sentence Detection language: bs edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_bs_3.2.0_3.0_1630317779410.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_bs_3.2.0_3.0_1630317779410.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "bs") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""Tražite sjajan izvor čitanja odlomaka na engleskom? Došli ste na pravo mjesto. Prema nedavnom istraživanju, navika čitanja u današnjoj mladosti brzo se smanjuje. Ne mogu se usredotočiti na dati odlomak za čitanje engleskog jezika duže od nekoliko sekundi! Takođe, čitanje je bilo i jeste sastavni dio svih takmičarskih ispita. Dakle, kako poboljšati svoje vještine čitanja? Odgovor na ovo pitanje zapravo je drugo pitanje: Kakva je korist od vještine čitanja? Glavna svrha čitanja je 'imati smisla'.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "bs") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("Tražite sjajan izvor čitanja odlomaka na engleskom? Došli ste na pravo mjesto. Prema nedavnom istraživanju, navika čitanja u današnjoj mladosti brzo se smanjuje. Ne mogu se usredotočiti na dati odlomak za čitanje engleskog jezika duže od nekoliko sekundi! Takođe, čitanje je bilo i jeste sastavni dio svih takmičarskih ispita. Dakle, kako poboljšati svoje vještine čitanja? Odgovor na ovo pitanje zapravo je drugo pitanje: Kakva je korist od vještine čitanja? Glavna svrha čitanja je 'imati smisla'.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('bs.sentence_detector').predict("Tražite sjajan izvor čitanja odlomaka na engleskom? Došli ste na pravo mjesto. Prema nedavnom istraživanju, navika čitanja u današnjoj mladosti brzo se smanjuje. Ne mogu se usredotočiti na dati odlomak za čitanje engleskog jezika duže od nekoliko sekundi! Takođe, čitanje je bilo i jeste sastavni dio svih takmičarskih ispita. Dakle, kako poboljšati svoje vještine čitanja? Odgovor na ovo pitanje zapravo je drugo pitanje: Kakva je korist od vještine čitanja? Glavna svrha čitanja je 'imati smisla'.", output_level ='sentence') ```

## Results ```bash +-----------------------------------------------------------------------------------------------+ |result | +-----------------------------------------------------------------------------------------------+ |[Tražite sjajan izvor čitanja odlomaka na engleskom?] | |[Došli ste na pravo mjesto.] | |[Prema nedavnom istraživanju, navika čitanja u današnjoj mladosti brzo se smanjuje.] | |[Ne mogu se usredotočiti na dati odlomak za čitanje engleskog jezika duže od nekoliko sekundi!]| |[Takođe, čitanje je bilo i jeste sastavni dio svih takmičarskih ispita.] | |[Dakle, kako poboljšati svoje vještine čitanja?] | |[Odgovor na ovo pitanje zapravo je drugo pitanje:] | |[Kakva je korist od vještine čitanja?] | |[Glavna svrha čitanja je 'imati smisla'.] | +-----------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|bs| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Italian BertForMaskedLM Base Cased model (from dbmdz) author: John Snow Labs name: bert_embeddings_base_italian_xxl_cased date: 2022-12-02 tags: [it, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-italian-xxl-cased` is a Italian model originally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_xxl_cased_it_4.2.4_3.0_1670017995735.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_xxl_cased_it_4.2.4_3.0_1670017995735.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_xxl_cased","it") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_xxl_cased","it") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_italian_xxl_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|415.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-xxl-cased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Telugu BertForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: bert_embeddings_indic_transformers date: 2022-12-06 tags: [te, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: te edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-te-bert` is a Telugu model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_te_4.2.4_3.0_1670326679548.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_indic_transformers_te_4.2.4_3.0_1670326679548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","te") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_indic_transformers","te") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_autotrain_ms_2_1174443640","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_autotrain_ms_2_1174443640","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_autotrain_ms_2_1174443640| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|916.0 MB| ## References - https://huggingface.co/benjamyu/autotrain-ms-2-1174443640 --- layout: model title: English BertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: bert_token_classifier_autotrain_final_784824206 date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-final-784824206` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_final_784824206_en_4.2.4_3.0_1669814522383.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_final_784824206_en_4.2.4_3.0_1669814522383.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_final_784824206","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_final_784824206","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autotrain_final_784824206| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-final-784824206 --- layout: model title: English BertForQuestionAnswering model (from jackh1995) author: John Snow Labs name: bert_qa_bert_finetuned_jackh1995 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned` is a English model orginally trained by `jackh1995`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_finetuned_jackh1995_en_4.0.0_3.0_1654534832705.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_finetuned_jackh1995_en_4.0.0_3.0_1654534832705.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_finetuned_jackh1995","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_finetuned_jackh1995","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_jackh1995").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_finetuned_jackh1995| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|381.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jackh1995/bert-finetuned --- layout: model title: Named Entity Recognition (NER) Model in Swedish (GloVe 840B 300) author: John Snow Labs name: swedish_ner_840B_300 date: 2020-08-30 task: Named Entity Recognition language: sv edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [ner, sv, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Swedish NER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. The model is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Product-`PRO`, Date-`DATE`, Event-`EVENT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_SV/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/swedish_ner_840B_300_sv_2.6.0_2.4_1598810268072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/swedish_ner_840B_300_sv_2.6.0_2.4_1598810268072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +------------------------+---------+ |chunk |ner_label| +------------------------+---------+ |William Henry Gates |PER | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |MISC | |Seattle |LOC | |Washington |LOC | |Gates Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |MISC | |Gates |MISC | |Gates |MISC | |Gates |MISC | |Microsoft |ORG | |Bill |MISC | |Melinda Gates Foundation|MISC | |Melinda Gates |MISC | |Ray Ozzie |PER | |Craig Mundie |PER | |Microsoft |ORG | +------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|swedish_ner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.6.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|sv| |Case sensitive:|false| {:.h2_title} ## Data Source Trained on a custom dataset with multi-lingual GloVe Embeddings ``glove_840B_300``. --- layout: model title: Part of Speech for Finnish author: John Snow Labs name: pos_ud_tdt date: 2020-05-04 23:32:00 +0800 task: Part of Speech Tagging language: fi edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, fi] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_tdt_fi_2.5.0_2.4_1588622348985.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_tdt_fi_2.5.0_2.4_1588622348985.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_tdt", "fi") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_tdt", "fi") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Sen lisäksi, että hän on pohjoisen kuningas, John Snow on englantilainen lääkäri ja johtava anestesian ja lääketieteellisen hygienian kehittämisessä."""] pos_df = nlu.load('fi.pos.ud_tdt').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=2, result='PRON', metadata={'word': 'Sen'}), Row(annotatorType='pos', begin=4, end=10, result='ADP', metadata={'word': 'lisäksi'}), Row(annotatorType='pos', begin=11, end=11, result='PUNCT', metadata={'word': ','}), Row(annotatorType='pos', begin=13, end=16, result='SCONJ', metadata={'word': 'että'}), Row(annotatorType='pos', begin=18, end=20, result='PRON', metadata={'word': 'hän'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_tdt| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|fi| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: PICO Classifier (BERT) author: John Snow Labs name: bert_sequence_classifier_pico_biobert date: 2022-02-07 tags: [bert, sequence_classification, en, licensed] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify medical text according to the PICO framework. This model is a [BioBERT-based](https://github.com/dmis-lab/biobert) classifier. ## Predicted Entities `CONCLUSIONS`, `DESIGN_SETTING`, `INTERVENTION`, `PARTICIPANTS`, `FINDINGS`, `MEASUREMENTS`, `AIMS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_pico_biobert_en_3.4.1_3.0_1644265236813.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_pico_biobert_en_3.4.1_3.0_1644265236813.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------------------------------------------------------------+------+ |text |result| +-------------------------------------------------------------------------------------------+------+ |To compare the results of recording enamel opacities using the TF and modified DDE indices.|[AIMS]| +-------------------------------------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_pico_biobert| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model is trained on a custom dataset derived from a PICO classification dataset. ## Benchmarking ```bash label precision recall f1-score support AIMS 0.92 0.94 0.93 3813 CONCLUSIONS 0.85 0.86 0.86 4314 DESIGN_SETTING 0.88 0.78 0.83 5628 FINDINGS 0.91 0.92 0.91 9242 INTERVENTION 0.71 0.78 0.74 2331 MEASUREMENTS 0.79 0.87 0.83 3219 PARTICIPANTS 0.86 0.81 0.83 2723 accuracy - - 0.86 31270 macro-avg 0.85 0.85 0.85 31270 weighted-avg 0.87 0.86 0.86 31270 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from benny6) author: John Snow Labs name: roberta_qa_tydi date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-tydiqa` is a English model originally trained by `benny6`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_tydi_en_4.3.0_3.0_1674222584111.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_tydi_en_4.3.0_3.0_1674222584111.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_tydi","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_tydi","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_tydi| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|471.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/benny6/roberta-tydiqa --- layout: model title: Legal Cooperation Clause Binary Classifier author: John Snow Labs name: legclf_cooperation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `cooperation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `cooperation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cooperation_clause_en_1.0.0_3.2_1660122299955.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cooperation_clause_en_1.0.0_3.2_1660122299955.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cooperation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[cooperation]| |[other]| |[other]| |[cooperation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cooperation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support cooperation 0.91 0.94 0.93 34 other 0.98 0.97 0.97 96 accuracy - - 0.96 130 macro-avg 0.95 0.95 0.95 130 weighted-avg 0.96 0.96 0.96 130 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_ali221000262 TFWav2Vec2ForCTC from ali221000262 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_ali221000262` is a English model originally trained by ali221000262. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262_en_4.2.0_3.0_1664036650196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262_en_4.2.0_3.0_1664036650196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_ali221000262| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Sentence Entity Resolver for billable ICD10-CM HCC codes author: John Snow Labs name: sbiobertresolve_icd10cm_augmented_billable_hcc date: 2021-02-06 task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, entity_resolution] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using chunk embeddings (augmented with synonyms, four times richer than previous resolver). It also adds support of 7-digit codes with HCC status. ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for `aux_label` parameter in the metadata to get HCC status. The HCC status can be divided to get further information: `billable status`, `hcc status`, and `hcc score`. For example, in the example shared `below the billable status is 1`, `hcc status is 1`, and `hcc score is 8`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_2.7.3_2.4_1612609178670.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_billable_hcc_en_2.7.3_2.4_1612609178670.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_icd10cm_augmented_billable_hcc``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. ```PROBLEM``` set in ```.setWhiteList()```.

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented_billable_hcc| |Compatibility:|Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| --- layout: model title: Detect PHI for Generic Deidentification in Romanian (BERT) author: John Snow Labs name: ner_deid_generic_bert date: 2022-08-15 tags: [licensed, clinical, ro, deidentification, phi, generic, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators to allow a generic model to be trained by using a Deep Learning architecture (Char CNN's - BiLSTM - CRF - word embeddings) inspired by a former state-of-the-art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It is trained with bert_base_cased embeddings and can detect 7 generic entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_4.0.2_3.0_1660551174367.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_bert_ro_4.0.2_3.0_1660551174367.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|LOCATION | |Drumul Oprea Nr |LOCATION | |972 |LOCATION | |Vaslui |LOCATION | |737405 |LOCATION | |+40(235)413773 |CONTACT | |25 May 2022 |DATE | |BUREAN MARIA |NAME | |77 |AGE | |Agota Evelyn Tımar |NAME | |2450502264401 |ID | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_bert| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.3 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.95 0.97 0.96 1186 CONTACT 0.99 0.98 0.98 366 DATE 0.96 0.92 0.94 4518 ID 1.00 1.00 1.00 679 LOCATION 0.91 0.90 0.90 1683 NAME 0.93 0.96 0.94 2916 PROFESSION 0.87 0.85 0.86 161 micro-avg 0.94 0.94 0.94 11509 macro-avg 0.94 0.94 0.94 11509 weighted-avg 0.95 0.94 0.94 11509 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from describeai) author: John Snow Labs name: t5_gemini_small date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gemini-small` is a English model originally trained by `describeai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_gemini_small_en_4.3.0_3.0_1675102559187.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_gemini_small_en_4.3.0_3.0_1675102559187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_gemini_small","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_gemini_small","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_gemini_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|916.0 MB| ## References - https://huggingface.co/describeai/gemini-small - https://www.describe-ai.com/gemini --- layout: model title: Fast Neural Machine Translation Model from Afrikaans to English author: John Snow Labs name: opus_mt_af_en date: 2021-06-01 tags: [open_source, seq2seq, translation, af, en, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: af target languages: en {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_af_en_xx_3.1.0_2.4_1622558064281.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_af_en_xx_3.1.0_2.4_1622558064281.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_af_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Japanese Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_bert_large_japanese_char_extended date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-japanese-char-extended` is a Japanese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_japanese_char_extended_ja_3.4.2_3.0_1649674799994.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_japanese_char_extended_ja_3.4.2_3.0_1649674799994.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_japanese_char_extended| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/KoichiYasuoka/bert-large-japanese-char-extended --- layout: model title: Javanese RoBERTa Embeddings (Small, IMDB Movie Review) author: John Snow Labs name: roberta_embeddings_javanese_roberta_small_imdb date: 2022-04-14 tags: [roberta, embeddings, jv, open_source] task: Embeddings language: jv edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `javanese-roberta-small-imdb` is a Javanese model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_javanese_roberta_small_imdb_jv_3.4.2_3.0_1649948176711.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_javanese_roberta_small_imdb_jv_3.4.2_3.0_1649948176711.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_javanese_roberta_small_imdb| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|468.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-roberta-small-imdb - https://arxiv.org/abs/1907.11692 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: Embeddings Clinical (Medium) author: John Snow Labs name: embeddings_clinical_medium date: 2023-04-07 tags: [licensed, en, clinical, embeddings] task: Embeddings language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on a list of clinical and biomedical datasets curated in-house, using the word2vec algorithm. The dataset curation cut-off date is March 2023 and the model is expected to have a better generalization on recent content. The size of the model is around 1 GB and has 200 dimensions. Our benchmark tests indicate that our legacy clinical embeddings (embeddings_clinical) can be replaced with this one while training a new model (existing/previous models will still need to use the legacy embeddings that they're trained with). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_medium_en_4.3.2_3.0_1680835759101.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_medium_en_4.3.2_3.0_1680835759101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium","en","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium","en","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|embeddings_clinical_medium| |Type:|embeddings| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|en| |Size:|787.5 MB| |Case sensitive:|true| |Dimension:|200| --- layout: model title: Swedish asr_wav2vec2_large_xlsr_swedish TFWav2Vec2ForCTC from marma author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_swedish date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_swedish` is a Swedish model originally trained by marma. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_swedish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_swedish_sv_4.2.0_3.0_1664118734633.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_swedish_sv_4.2.0_3.0_1664118734633.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_swedish', lang = 'sv') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_swedish", lang = "sv") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_swedish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|sv| |Size:|756.1 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Drug Substance to UMLS Code Pipeline author: John Snow Labs name: umls_drug_substance_resolver_pipeline date: 2023-04-11 tags: [licensed, clinical, en, umls, pipeline, drug, subtance] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities (Drug Substances) with their corresponding UMLS CUI codes. You’ll just feed your text and it will return the corresponding UMLS codes. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_drug_substance_resolver_pipeline_en_4.3.2_3.0_1681217098344.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_drug_substance_resolver_pipeline_en_4.3.2_3.0_1681217098344.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("umls_drug_substance_resolver_pipeline", "en", "clinical/models") result = pipeline.annotate("The patient was given metformin, lenvatinib and Magnesium hydroxide 100mg/1ml") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = PretrainedPipeline("umls_drug_substance_resolver_pipeline", "en", "clinical/models") val result = pipeline.annotate("The patient was given metformin, lenvatinib and Magnesium hydroxide 100mg/1ml") ``` {:.nlu-block} ```python +-----------------------------+---------+---------+ |chunk |ner_label|umls_code| +-----------------------------+---------+---------+ |metformin |DRUG |C0025598 | |lenvatinib |DRUG |C2986924 | |Magnesium hydroxide 100mg/1ml|DRUG |C1134402 | +-----------------------------+---------+---------+ ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_drug_substance_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|5.1 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: Multilingual DistilBertForQuestionAnswering Base Cased model (from monakth) author: John Snow Labs name: distilbert_qa_monakth_base_case_finetuned_squad date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multilingual-cased-finetuned-squad` is a Multilingual model originally trained by `monakth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_monakth_base_case_finetuned_squad_xx_4.3.0_3.0_1672767194965.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_monakth_base_case_finetuned_squad_xx_4.3.0_3.0_1672767194965.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_monakth_base_case_finetuned_squad","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_monakth_base_case_finetuned_squad","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_monakth_base_case_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/monakth/distilbert-base-multilingual-cased-finetuned-squad --- layout: model title: Pipeline to Extract Pharmacological Entities From Spanish Medical Texts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_pharmacology_pipeline date: 2023-03-20 tags: [es, clinical, licensed, token_classification, bert, ner, pharmacology] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_pharmacology](https://nlp.johnsnowlabs.com/2022/08/11/bert_token_classifier_pharmacology_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_pharmacology_pipeline_es_4.3.0_3.2_1679298404485.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_pharmacology_pipeline_es_4.3.0_3.2_1679298404485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_pharmacology_pipeline", "es", "clinical/models") text = '''Se realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_pharmacology_pipeline", "es", "clinical/models") val text = "Se realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa)." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------|--------:|------:|:--------------|-------------:| | 0 | creatinkinasa | 32 | 44 | PROTEINAS | 0.999973 | | 1 | LDH | 54 | 56 | PROTEINAS | 0.999972 | | 2 | urea | 66 | 69 | NORMALIZABLES | 0.999977 | | 3 | CA 19.9 | 81 | 87 | PROTEINAS | 0.999964 | | 4 | vimentina | 139 | 147 | PROTEINAS | 0.999961 | | 5 | S-100 | 150 | 154 | PROTEINAS | 0.999861 | | 6 | HMB-45 | 157 | 162 | PROTEINAS | 0.999965 | | 7 | actina | 166 | 171 | PROTEINAS | 0.999967 | | 8 | Cisplatino | 220 | 229 | NORMALIZABLES | 0.999988 | | 9 | Interleukina II | 232 | 246 | PROTEINAS | 0.999965 | | 10 | Dacarbacina | 249 | 259 | NORMALIZABLES | 0.999988 | | 11 | Interferon alfa | 263 | 277 | PROTEINAS | 0.999961 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_pharmacology_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|410.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: XLNet Large CoNLL-03 NER Pipeline author: John Snow Labs name: xlnet_large_token_classifier_conll03_pipeline date: 2022-06-24 tags: [open_source, ner, token_classifier, xlnet, conll03, large, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlnet_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/28/xlnet_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlnet_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1656081418731.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlnet_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1656081418731.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlnet_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("xlnet_large_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PERSON | |John Snow Labs|ORG | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlnet_large_token_classifier_conll03_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|32.1 KB| ## Included Models - DocumentAssembler - TokenizerModel - NormalizerModel --- layout: model title: Detect PHI for Deidentification purposes (Spanish, Roberta, augmented) author: John Snow Labs name: ner_deid_subentity_roberta_augmented date: 2022-02-15 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities, which is more than the previously released `ner_deid_subentity_roberta` model. This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf and MeddoCan datasets, and includes several data augmentation mechanisms. This is a version that includes Roberta Clinical embeddings. You can find as well `ner_deid_subentity_augmented` that uses Sciwi 300d embeddings based instead of Roberta. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `ID`, `STREET`, `USERNAME`, `SEX`, `EMAIL`, `ZIP`, `MEDICALRECORD`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_roberta_augmented_es_3.3.4_2.4_1644927666923.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_roberta_augmented_es_3.3.4_2.4_1644927666923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") roberta_embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_subentity_roberta_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] df = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(df).transform(df) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val roberta_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_deid_subentity_roberta_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val df = Seq(text).toDF("text") val results = pipeline.fit(df).transform(df) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.subentity.roberta").predict(""" Antonio Miguel Martínez, varón de de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-PATIENT| | Miguel| I-PATIENT| | Martínez| I-PATIENT| | ,| O| | varón| B-SEX| | de| O| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-CITY| | ,| O| | España| B-COUNTRY| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19| O| | el| O| | dia| O| | 14| B-DATE| | de| I-DATE| | Marzo| I-DATE| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| O| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-HOSPITAL| | San| I-HOSPITAL| | Carlos| I-HOSPITAL| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_roberta_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|16.3 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 1874.0 165.0 186.0 2060.0 0.9191 0.9097 0.9144 HOSPITAL 241.0 19.0 54.0 295.0 0.9269 0.8169 0.8685 DATE 954.0 17.0 15.0 969.0 0.9825 0.9845 0.9835 ORGANIZATION 2521.0 483.0 468.0 2989.0 0.8392 0.8434 0.8413 CITY 1464.0 369.0 289.0 1753.0 0.7987 0.8351 0.8165 ID 35.0 1.0 0.0 35.0 0.9722 1.0 0.9859 STREET 194.0 8.0 6.0 200.0 0.9604 0.97 0.9652 USERNAME 7.0 0.0 4.0 11.0 1.0 0.6364 0.7778 SEX 618.0 9.0 9.0 627.0 0.9856 0.9856 0.9856 EMAIL 134.0 0.0 0.0 134.0 1.0 1.0 1.0 ZIP 138.0 0.0 1.0 139.0 1.0 0.9928 0.9964 MEDICALRECORD 29.0 10.0 0.0 29.0 0.7436 1.0 0.8529 PROFESSION 231.0 20.0 30.0 261.0 0.9203 0.8851 0.9023 PHONE 44.0 0.0 6.0 50.0 1.0 0.88 0.9362 COUNTRY 458.0 96.0 103.0 561.0 0.8267 0.8164 0.8215 DOCTOR 432.0 38.0 48.0 480.0 0.9191 0.9 0.9095 AGE 509.0 9.0 10.0 519.0 0.9826 0.9807 0.9817 macro - - - - - - 0.9141 micro - - - - - - 0.8891 ``` --- layout: model title: English Deberta Embeddings model (from scales-okn) author: John Snow Labs name: deberta_embeddings_docket_language_model date: 2023-03-13 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, en, tensorflow] task: Embeddings language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `docket-language-model` is a English model originally trained by `scales-okn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_docket_language_model_en_4.3.1_3.0_1678704745796.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_docket_language_model_en_4.3.1_3.0_1678704745796.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_docket_language_model","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_docket_language_model","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_docket_language_model| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|false| ## References https://huggingface.co/scales-okn/docket-language-model --- layout: model title: Detect Person, Location, Organization, and Miscellaneous entities in Arabic (ANERcorp) author: John Snow Labs name: aner_cc_300d date: 2022-07-26 tags: [ner, ar, open_source] task: Named Entity Recognition language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses Arabic word embeddings to find 4 different types of entities in Arabic text. It is trained using `arabic_w2v_cc_300d` word embeddings, so please use the same embeddings in the pipeline. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/aner_cc_300d_ar_4.0.0_3.0_1658869537384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/aner_cc_300d_ar_4.0.0_3.0_1658869537384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python word_embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("aner_cc_300d", "ar") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز") ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("aner_cc_300d", "ar") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.ner").predict("""في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز""") ```

## Results ```bash | | ner_chunk | entity | |---:|-------------------------:|-------------:| | 0 | قوات الثورة العربية | ORG | | 1 | دمشق | LOC | | 2 | الإنكليز | PER | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|aner_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|ar| |Size:|14.9 MB| |Dependencies:|arabic_w2v_cc_300d| ## References This model is trained on data obtained from http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp ## Benchmarking ```bash label tp fp fn prec rec f1 B-LOC 163 28 34 0.853403 0.827411 0.840206 I-ORG 60 10 5 0.857142 0.923077 0.888889 I-MIS 124 53 53 0.700565 0.700565 0.700565 I-LOC 64 20 23 0.761904 0.735632 0.748538 B-MIS 297 71 52 0.807065 0.851003 0.828452 I-PER 84 23 13 0.785046 0.865979 0.823530 B-ORG 54 9 12 0.857142 0.818181 0.837210 B-PER 182 26 33 0.875 0.846512 0.860520 Macro-average 1028 240 225 0.812159 0.821045 0.816578 Micro-average 1028 240 225 0.810726 0.820431 0.815550 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from hark99) author: John Snow Labs name: distilbert_qa_hark99_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `hark99`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_hark99_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725361304.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_hark99_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725361304.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hark99_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hark99_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_hark99").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hark99_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hark99/distilbert-base-uncased-finetuned-squad --- layout: model title: French CamemBert Embeddings (from peterhsu) author: John Snow Labs name: camembert_embeddings_peterhsu_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `peterhsu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_peterhsu_generic_model_fr_3.4.4_3.0_1653989980368.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_peterhsu_generic_model_fr_3.4.4_3.0_1653989980368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_peterhsu_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/peterhsu/dummy-model --- layout: model title: Recognize Entities OntoNotes - ELECTRA Base author: John Snow Labs name: onto_recognize_entities_electra_base date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [en, open_source, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `electra_base_uncased` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_base_en_2.7.0_2.4_1607511462448.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_base_en_2.7.0_2.4_1607511462448.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_electra_base') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_electra_base") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu text = ["""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament."""] ner_df = nlu.load('en.ner.onto.electra.base').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |Parliament |ORG | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | |Parliament |ORG | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_electra_base| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl36 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl36` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl36_en_4.3.0_3.0_1675122498742.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl36_en_4.3.0_3.0_1675122498742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl36","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl36","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl36| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|572.2 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl36 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English DistilBertForQuestionAnswering Cased model (from minhdang241) author: John Snow Labs name: distilbert_qa_robust_tapt date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robustqa-tapt` is a English model originally trained by `minhdang241`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_robust_tapt_en_4.3.0_3.0_1672775384901.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_robust_tapt_en_4.3.0_3.0_1672775384901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_robust_tapt","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_robust_tapt","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_robust_tapt| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/minhdang241/robustqa-tapt --- layout: model title: Relation Extraction between different oncological entity types using granular classes (ReDL) author: John Snow Labs name: redl_oncology_granular_biobert_wip date: 2023-01-15 tags: [licensed, clinical, oncology, en, relation_extraction, temporal, test, biomarker, anatomy, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Using this relation extraction model, four relation types can be identified: is_date_of (between date entities and other clinical entities), is_size_of (between Tumor_Finding and Tumor_Size), is_location_of (between anatomical entities and other entities) and is_finding_of (between test entities and their results). ## Predicted Entities `is_date_of`, `is_finding_of`, `is_location_of`, `is_size_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_granular_biobert_wip_en_4.2.4_3.0_1673768709402.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_granular_biobert_wip_en_4.2.4_3.0_1673768709402.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash +----------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ | relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +----------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ |is_date_of|Cancer_Surgery| 2| 11|mastectomy|Relative_Date| 27| 40|two months ago| 0.9652523| |is_size_of| Tumor_Size| 49| 52| 3 cm|Tumor_Finding| 54| 57| mass|0.81723577| +----------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_granular_biobert_wip| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.83 0.91 0.87 is_date_of 0.82 0.80 0.81 is_finding_of 0.92 0.85 0.88 is_location_of 0.95 0.85 0.90 is_size_of 0.91 0.80 0.85 macro-avg 0.89 0.84 0.86 ``` --- layout: model title: Detect Entities Related to Cancer Diagnosis author: John Snow Labs name: ner_oncology_diagnosis date: 2022-11-24 tags: [licensed, clinical, en, ner, oncology] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to cancer diagnosis, such as Metastasis, Histological_Type or Invasion. Definitions of Predicted Entities: - `Adenopathy`: Mentions of pathological findings of the lymph nodes. - `Cancer_Dx`: Mentions of cancer diagnoses (such as "breast cancer") or pathological types that are usually used as synonyms for "cancer" (e.g. "carcinoma"). When anatomical references are present, they are included in the Cancer_Dx extraction. - `Cancer_Score`: Clinical or imaging scores that are specific for cancer settings (e.g. "BI-RADS" or "Allred score"). - `Grade`: All pathological grading of tumors (e.g. "grade 1") or degrees of cellular differentiation (e.g. "well-differentiated") - `Histological_Type`: Histological variants or cancer subtypes, such as "papillary", "clear cell" or "medullary". - `Invasion`: Mentions that refer to tumor invasion, such as "invasion" or "involvement". Metastases or lymph node involvement are excluded from this category. - `Metastasis`: Terms that indicate a metastatic disease. Anatomical references are not included in these extractions. - `Pathology_Result`: The findings of a biopsy from the pathology report that is not covered by another entity (e.g. "malignant ductal cells"). - `Performance_Status`: Mentions of performance status scores, such as ECOG and Karnofsky. The name of the score is extracted together with the result (e.g. "ECOG performance status of 4"). - `Staging`: Mentions of cancer stage such as "stage 2b" or "T2N1M0". It also includes words such as "in situ", "early-stage" or "advanced". - `Tumor_Finding`: All nonspecific terms that may be related to tumors, either malignant or benign (for example: "mass", "tumor", "lesion", or "neoplasm"). - `Tumor_Size`: Size of the tumor, including numerical value and unit of measurement (e.g. "3 cm"). ## Predicted Entities `Adenopathy`, `Cancer_Dx`, `Cancer_Score`, `Grade`, `Histological_Type`, `Invasion`, `Metastasis`, `Pathology_Result`, `Performance_Status`, `Staging`, `Tumor_Finding`, `Tumor_Size` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_diagnosis_en_4.2.2_3.0_1669300474926.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_diagnosis_en_4.2.2_3.0_1669300474926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------|:------------------| | tumor | Tumor_Finding | | adenopathies | Adenopathy | | invasive | Histological_Type | | ductal | Histological_Type | | carcinoma | Cancer_Dx | | metastasis | Metastasis | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_diagnosis| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Histological_Type 354 63 99 453 0.85 0.78 0.81 Staging 234 27 24 258 0.90 0.91 0.90 Cancer_Score 36 15 26 62 0.71 0.58 0.64 Tumor_Finding 1121 83 136 1257 0.93 0.89 0.91 Invasion 154 27 27 181 0.85 0.85 0.85 Tumor_Size 1058 126 71 1129 0.89 0.94 0.91 Adenopathy 66 10 30 96 0.87 0.69 0.77 Performance_Status 116 15 19 135 0.89 0.86 0.87 Pathology_Result 852 686 290 1142 0.55 0.75 0.64 Metastasis 356 15 14 370 0.96 0.96 0.96 Cancer_Dx 1302 88 92 1394 0.94 0.93 0.94 Grade 201 23 35 236 0.90 0.85 0.87 macro_avg 5850 1178 863 6713 0.85 0.83 0.84 micro_avg 5850 1178 863 6713 0.85 0.87 0.86 ``` --- layout: model title: Part of Speech for Chinese author: John Snow Labs name: pos_ud_gsd_trad date: 2021-03-09 tags: [part_of_speech, open_source, chinese, pos_ud_gsd_trad, zh] task: Part of Speech Tagging language: zh edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - AUX - ADJ - PUNCT - ADV - VERB - NUM - NOUN - PRON - PART - ADP - DET - CCONJ - PROPN - X - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_trad_zh_3.0.0_3.0_1615292436582.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_trad_zh_3.0.0_3.0_1615292436582.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_gsd_trad", "zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['从John Snow Labs你好！ ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_gsd_trad", "zh") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("从John Snow Labs你好！ ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""从John Snow Labs你好！ ""] token_df = nlu.load('zh.pos.ud_gsd_trad').predict(text) token_df ```

## Results ```bash token pos 0 从 PROPN 1 JohnSnowLabs X 2 你 PRON 3 好 ADJ 4 ！ PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd_trad| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|zh| --- layout: model title: English DistilBertForQuestionAnswering model (from Ayoola) author: John Snow Labs name: distilbert_qa_Ayoola_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Ayoola`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Ayoola_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724089038.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Ayoola_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724089038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Ayoola_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Ayoola_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Ayoola").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Ayoola_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Ayoola/distilbert-base-uncased-finetuned-squad --- layout: model title: Swedish BERT Base Cased Embedding author: John Snow Labs name: bert_base_cased date: 2021-09-07 tags: [open_source, bert_embeddings, swedish, cased, sv] task: Embeddings language: sv edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on aproximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_sv_3.2.2_3.0_1630999671555.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_sv_3.2.2_3.0_1630999671555.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_cased", "sv") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_cased", "sv") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("sv.embed.bert.base_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_cased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|sv| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/KB/bert-base-swedish-cased --- layout: model title: Legal No violation Clause Binary Classifier author: John Snow Labs name: legclf_no_violation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-violation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-violation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_violation_clause_en_1.0.0_3.2_1660122713710.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_violation_clause_en_1.0.0_3.2_1660122713710.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_violation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-violation]| |[other]| |[other]| |[no-violation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_violation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-violation 1.00 1.00 1.00 35 other 1.00 1.00 1.00 93 accuracy - - 1.00 128 macro-avg 1.00 1.00 1.00 128 weighted-avg 1.00 1.00 1.00 128 ``` --- layout: model title: Fast Neural Machine Translation Model from Lunda to English author: John Snow Labs name: opus_mt_lun_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, lun, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `lun` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_lun_en_xx_2.7.0_2.4_1609167008180.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_lun_en_xx_2.7.0_2.4_1609167008180.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_lun_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Portuguese BertForMaskedLM Cased model (from pucpr) author: John Snow Labs name: bert_embeddings_biobertpt_all date: 2022-12-02 tags: [pt, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: pt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biobertpt-all` is a Portuguese model originally trained by `pucpr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_biobertpt_all_pt_4.2.4_3.0_1670020710320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_biobertpt_all_pt_4.2.4_3.0_1670020710320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_biobertpt_all","pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_biobertpt_all","pt") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_biobertpt_all| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|667.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/pucpr/biobertpt-all - https://www.aclweb.org/anthology/2020.clinicalnlp-1.7/ - https://www.aclweb.org/anthology/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: Detect Entities Related to Cancer Therapies author: John Snow Labs name: ner_oncology_therapy date: 2022-11-24 tags: [clinical, en, licensed, oncology, treatment, ner] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to oncology therapies using granular labels, including mentions of treatments, posology information and line of therapy. Definitions of Predicted Entities: - `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. - `Chemotherapy`: Mentions of chemotherapy drugs, or unspecific words such as "chemotherapy". - `Cycle_Count`: The total number of cycles being administered of an oncological therapy (e.g. "5 cycles"). - `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. "day 5"). - `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. "third cycle"). - `Dosage`: The quantity prescribed by the physician for an active ingredient. - `Duration`: Words indicating the duration of a treatment (e.g. "for 2 weeks"). - `Frequency`: Words indicating the frequency of treatment administration (e.g. "daily" or "bid"). - `Hormonal_Therapy`: Mentions of hormonal drugs used to treat cancer, or unspecific words such as "hormonal therapy". - `Immunotherapy`: Mentions of immunotherapy drugs, or unspecific words such as "immunotherapy". - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Radiotherapy`: Terms that indicate the use of Radiotherapy. - `Radiation_Dose`: Dose used in radiotherapy. - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Route`: Words indicating the type of administration route (such as "PO" or "transdermal"). - `Targeted_Therapy`: Mentions of targeted therapy drugs, or unspecific words such as "targeted therapy". - `Unspecific_Therapy`: Terms that indicate a known cancer therapy but that is not specific to any other therapy entity (e.g. "chemoradiotherapy" or "adjuvant therapy"). ## Predicted Entities `Cancer_Surgery`, `Chemotherapy`, `Cycle_Count`, `Cycle_Day`, `Cycle_Number`, `Dosage`, `Duration`, `Frequency`, `Hormonal_Therapy`, `Immunotherapy`, `Line_Of_Therapy`, `Radiotherapy`, `Radiation_Dose`, `Response_To_Treatment`, `Route`, `Targeted_Therapy`, `Unspecific_Therapy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_en_4.2.2_3.0_1669308088671.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_en_4.2.2_3.0_1669308088671.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------------------------|:----------------------| | mastectomy | Cancer_Surgery | | axillary lymph node dissection | Cancer_Surgery | | radiotherapy | Radiotherapy | | recurred | Response_To_Treatment | | adriamycin | Chemotherapy | | 60 mg/m2 | Dosage | | cyclophosphamide | Chemotherapy | | 600 mg/m2 | Dosage | | six courses | Cycle_Count | | first line | Line_Of_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_therapy| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Cycle_Number 78 41 19 97 0.66 0.80 0.72 Response_To_Treatment 451 205 145 596 0.69 0.76 0.72 Cycle_Count 210 75 20 230 0.74 0.91 0.82 Unspecific_Therapy 189 76 89 278 0.71 0.68 0.70 Chemotherapy 831 87 48 879 0.91 0.95 0.92 Targeted_Therapy 194 28 34 228 0.87 0.85 0.86 Radiotherapy 279 35 31 310 0.89 0.90 0.89 Cancer_Surgery 720 192 99 819 0.79 0.88 0.83 Line_Of_Therapy 95 6 11 106 0.94 0.90 0.92 Hormonal_Therapy 170 6 15 185 0.97 0.92 0.94 Immunotherapy 96 17 32 128 0.85 0.75 0.80 Cycle_Day 205 38 43 248 0.84 0.83 0.84 Frequency 363 33 64 427 0.92 0.85 0.88 Route 93 6 20 113 0.94 0.82 0.88 Duration 527 102 234 761 0.84 0.69 0.76 Dosage 959 63 101 1060 0.94 0.90 0.92 Radiation_Dose 106 12 20 126 0.90 0.84 0.87 macro_avg 5566 1022 1025 6591 0.85 0.84 0.84 micro_avg 5566 1022 1025 6591 0.85 0.84 0.84 ``` --- layout: model title: Translate Setswana to English Pipeline author: John Snow Labs name: translate_tn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, tn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `tn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_tn_en_xx_2.7.0_2.4_1609686822973.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_tn_en_xx_2.7.0_2.4_1609686822973.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_tn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_tn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.tn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_tn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_base_patch16_224_in21k_aidSat ViTForImageClassification from YKXBCi author: John Snow Labs name: image_classifier_vit_base_patch16_224_in21k_aidSat date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_in21k_aidSat` is a English model originally trained by YKXBCi. ## Predicted Entities `Square`, `Farmland`, `BaseballField`, `Park`, `Commercial`, `Pond`, `Airport`, `SparseResidential`, `Church`, `School`, `Viaduct`, `Stadium`, `Desert`, `BareLand`, `MediumResidential`, `Center`, `Industrial`, `Playground`, `Port`, `DenseResidential`, `StorageTanks`, `Beach`, `Bridge`, `Mountain`, `River`, `Meadow`, `Resort`, `Parking`, `Forest`, `RailwayStation` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_aidSat_en_4.1.0_3.0_1660167644527.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_aidSat_en_4.1.0_3.0_1660167644527.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_in21k_aidSat", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_in21k_aidSat", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_in21k_aidSat| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.0 MB| --- layout: model title: Arabic Part of Speech Tagger (DA-Dialectal Arabic dataset, Modern Standard Arabic-MSA POS) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_da_pos_msa date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-da-pos-msa` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_msa_ar_3.4.2_3.0_1650993280099.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_msa_ar_3.4.2_3.0_1650993280099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_da_pos_msa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-da-pos-msa - https://dl.acm.org/doi/pdf/10.5555/1621804.1621808 - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from hamishm) author: John Snow Labs name: distilbert_qa_hamishm_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `hamishm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_hamishm_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771049910.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_hamishm_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771049910.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hamishm_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hamishm_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hamishm_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hamishm/distilbert-base-uncased-finetuned-squad --- layout: model title: Detect diseases in medical text (biobert) author: John Snow Labs name: ner_diseases_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract entities pertaining to different types of general diseases using pretrained NER model. ## Predicted Entities `Disease` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DIAG_PROC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_en_3.0.0_3.0_1617260638998.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diseases_biobert_en_3.0.0_3.0_1617260638998.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_diseases_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_diseases_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.diseases.biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_diseases_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: Czech asr_wav2vec2_xls_r_300m_250 TFWav2Vec2ForCTC from comodoro author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_250 date: 2022-09-25 tags: [wav2vec2, cs, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: cs edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_250` is a Czech model originally trained by comodoro. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_250_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_250_cs_4.2.0_3.0_1664119400609.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_250_cs_4.2.0_3.0_1664119400609.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_250', lang = 'cs') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_250", lang = "cs") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_250| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|cs| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from linh101201) author: John Snow Labs name: roberta_qa_linh101201_base_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `linh101201`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_linh101201_base_finetuned_squad_en_4.3.0_3.0_1674217360099.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_linh101201_base_finetuned_squad_en_4.3.0_3.0_1674217360099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_linh101201_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_linh101201_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_linh101201_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|424.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/linh101201/roberta-base-finetuned-squad --- layout: model title: English XLMRobertaForTokenClassification Large Uncased model (from asahi417) author: John Snow Labs name: xlmroberta_ner_tner_large_uncased_ontonotes5 date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tner-xlm-roberta-large-uncased-ontonotes5` is a English model originally trained by `asahi417`. ## Predicted Entities `language`, `time`, `percent`, `quantity`, `product`, `ordinal number`, `cardinal number`, `event`, `geopolitical area`, `facility`, `organization`, `work of art`, `group`, `money`, `law`, `person`, `location`, `date` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_uncased_ontonotes5_en_4.1.0_3.0_1660425481816.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_uncased_ontonotes5_en_4.1.0_3.0_1660425481816.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_uncased_ontonotes5","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_uncased_ontonotes5","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tner_large_uncased_ontonotes5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|false| |Max sentence length:|256| ## References - https://huggingface.co/asahi417/tner-xlm-roberta-large-uncased-ontonotes5 - https://github.com/asahi417/tner --- layout: model title: English asr_wav2vec2_xls_r_timit_tokenizer_base TFWav2Vec2ForCTC from hrdipto author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_timit_tokenizer_base` is a English model originally trained by hrdipto. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base_en_4.2.0_3.0_1664040322145.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base_en_4.2.0_3.0_1664040322145.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_timit_tokenizer_base| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Japanese Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_base_japanese_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, ja, open_source] task: Part of Speech Tagging language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-upos` is a Japanese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_japanese_upos_ja_3.4.2_3.0_1652091854179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_japanese_upos_ja_3.4.2_3.0_1652091854179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_japanese_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ja| |Size:|338.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-base-japanese-upos - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from vinitharaj) author: John Snow Labs name: distilbert_qa_vinitharaj_base_uncased_finetuned_squad2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad2` is a English model originally trained by `vinitharaj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_vinitharaj_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773632350.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_vinitharaj_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773632350.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vinitharaj_base_uncased_finetuned_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vinitharaj_base_uncased_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vinitharaj_base_uncased_finetuned_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vinitharaj/distilbert-base-uncased-finetuned-squad2 --- layout: model title: English BertForQuestionAnswering model (from AnonymousSub) author: John Snow Labs name: bert_qa_bert_FT_new_newsqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_FT_new_newsqa` is a English model orginally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_FT_new_newsqa_en_4.0.0_3.0_1654185046001.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_FT_new_newsqa_en_4.0.0_3.0_1654185046001.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_FT_new_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_FT_new_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.bert.ft_new.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_FT_new_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/bert_FT_new_newsqa --- layout: model title: Fast Neural Machine Translation Model from Luo (Kenya and Tanzania) to English author: John Snow Labs name: opus_mt_luo_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, luo, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `luo` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_luo_en_xx_2.7.0_2.4_1609167362702.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_luo_en_xx_2.7.0_2.4_1609167362702.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_luo_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Vietnamese BertForQuestionAnswering model (from nvkha) author: John Snow Labs name: bert_qa_bert_qa_vi_nvkha date: 2022-06-03 tags: [open_source, question_answering, bert] task: Question Answering language: vi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-qa-vi` is a Vietnamese model orginally trained by `nvkha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_qa_vi_nvkha_vi_4.0.0_3.0_1654249815695.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_qa_vi_nvkha_vi_4.0.0_3.0_1654249815695.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_qa_vi_nvkha","vi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_qa_vi_nvkha","vi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("vi.answer_question.bert.by_nvkha").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_qa_vi_nvkha| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|vi| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nvkha/bert-qa-vi --- layout: model title: Translate Tongan to English Pipeline author: John Snow Labs name: translate_to_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, to, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `to` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_to_en_xx_2.7.0_2.4_1609688214187.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_to_en_xx_2.7.0_2.4_1609688214187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_to_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_to_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.to.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_to_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Summarize Clinical Notes in Layman Terms author: John Snow Labs name: summarizer_clinical_laymen date: 2023-05-31 tags: [licensed, en, clinical, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based summarization model that is finetuned with custom dataset by John Snow Labs to avoid using clinical jargon on the summaries. It can generate summaries up to 512 tokens given an input text (max 1024 tokens). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_en_4.4.2_3.0_1685557633038.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_laymen_en_4.4.2_3.0_1685557633038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['This is a clinical note about a 34-year-old woman who is interested in having weight loss surgery. She has been overweight for over 20 years and wants to have more energy and improve her self-image. She has tried many diets and weight loss programs, but has not been successful in keeping the weight off. She has a history of hypertension and shortness of breath, but is not allergic to any medications. She will have an upper endoscopy and will be contacted by a nutritionist and social worker. The plan is to have her weight loss surgery through the gastric bypass, rather than Lap-Band.'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_laymen| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.5 MB| --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from danhsf) author: John Snow Labs name: xlmroberta_ner_danhsf_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `danhsf`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_danhsf_base_finetuned_panx_de_4.1.0_3.0_1660432036772.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_danhsf_base_finetuned_panx_de_4.1.0_3.0_1660432036772.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_danhsf_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_danhsf_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_danhsf_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/danhsf/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal Investment Advisory Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_investment_advisory_agreement date: 2022-12-06 tags: [en, legal, classification, agreement, investment, advisory, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_investment_advisory_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `investment-advisory-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `investment-advisory-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_investment_advisory_agreement_en_1.0.0_3.0_1670357942473.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_investment_advisory_agreement_en_1.0.0_3.0_1670357942473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[investment-advisory-agreement]| |[other]| |[other]| |[investment-advisory-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_investment_advisory_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support investment-advisory-agreement 0.97 0.98 0.98 60 other 0.99 0.98 0.99 111 accuracy - - 0.98 171 macro-avg 0.98 0.98 0.98 171 weighted-avg 0.98 0.98 0.98 171 ``` --- layout: model title: Visual NER - CORD (Receipts) author: John Snow Labs name: visualner_receipts date: 2022-09-21 tags: [xx, licensed] task: OCR Object Detection language: xx edition: Visual NLP 4.0.0 spark_version: 3.2 supported: true annotator: VisualDocumentNERv21 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Visual NER, a model trained on the top of LayoutLMV2 to detect regions in Tickets. This model can be used after, for example, the Binary Image Classifier of Tickets, available at https://nlp.johnsnowlabs.com/2022/09/07/finvisualclf_vit_tickets_en.html ## Predicted Entities `COMPANY`, `DATE`, `AMOUNT`, `NAME`, `NUM`, `UNITPRICE`, `CNT`, `DISCOUNTPRICE`, `PRICE`, `ITEMSUBTOTAL`, `VATyn`, `SUBTOTAL`, `TOTALDISCOUNT`, `SERVICEPRICE`, `OTHERSVCPRICE`, `TAX`, `TOTAL`, `CASH`, `CHANGE`, `CREDITCARD`, `EMONEY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/visualner_receipts_xx_4.0.0_3.2_1663753935456.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/visualner_receipts_xx_4.0.0_3.2_1663753935456.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage()\ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) img_to_hocr = ImageToHocr()\ .setInputCol("image")\ .setOutputCol("hocr")\ .setIgnoreResolution(False)\ .setOcrParams(["preserve_interword_spaces=0"]) tokenizer = HocrTokenizer()\ .setInputCol("hocr")\ .setOutputCol("token") doc_ner = VisualDocumentNerV21()\ .pretrained("visualner_receipts", "en", "clinical/ocr")\ .setInputCols(["token", "image"])\ .setOutputCol("entities") draw = ImageDrawAnnotations() \ .setInputCol("image") \ .setInputChunksCol("entities") \ .setOutputCol("image_with_annotations") \ .setFontSize(10) \ .setLineWidth(4)\ .setRectColor(Color.red) # OCR pipeline pipeline = PipelineModel(stages=[ binary_to_image, img_to_hocr, tokenizer, doc_ner, draw ]) import pkg_resources bin_df = spark.read.format("binaryFile").load('data/t01.jpg') bin_df.show() results = pipeline.transform(bin_df).cache() res = results.collect() ## since pyspark2.3 doesn't have element_at, 'getItem' is involked path_array = f.split(results['path'], '/') # from pyspark2.4 # results.withColumn("filename", f.element_at(f.split("path", "/"), -1)) \ results.withColumn('filename', path_array.getItem(f.size(path_array)- 1)) \ .withColumn("exploded_entities", f.explode("entities")) \ .select("filename", "exploded_entities") \ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_extractive","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_extractive","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_extractive| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/autoevaluate/extractive-question-answering --- layout: model title: BERTje A Dutch BERT model author: John Snow Labs name: bert_base_dutch_cased date: 2021-05-20 tags: [open_source, embeddings, bert, dutch, nl] task: Embeddings language: nl edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BERTje is a Dutch pre-trained BERT model developed at the University of Groningen. For details, check out our paper on [arXiv](https://arxiv.org/abs/1912.09582), the code on [Github](https://github.com/wietsedv/bertje) and related work on [Semantic Scholar](https://www.semanticscholar.org/paper/BERTje%3A-A-Dutch-BERT-Model-Vries-Cranenburgh/a4d5e425cac0bf84c86c0c9f720b6339d6288ffa). The paper and Github page mention fine-tuned models that are available [here](https://huggingface.co/wietsedv). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_dutch_cased_nl_3.1.0_3.0_1621500934814.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_dutch_cased_nl_3.1.0_3.0_1621500934814.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_dutch_cased", "nl") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_dutch_cased", "nl") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("nl.embed.bert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_dutch_cased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|nl| |Case sensitive:|true| ## Data Source https://huggingface.co/dbmdz/bert-base-german-cased ## Benchmarking ```bash The arXiv paper lists benchmarks. Here are a couple of comparisons between BERTje, multilingual BERT, BERT-NL, and RobBERT that were done after writing the paper. Unlike some other comparisons, the fine-tuning procedures for these benchmarks are identical for each pre-trained model. You may be able to achieve higher scores for individual models by optimizing fine-tuning procedures. More experimental results will be added to this page when they are finished. Technical details about how a fine-tuned these models will be published later as well as downloadable fine-tuned checkpoints. All of the tested models are *base* sized (12) layers with cased tokenization. Headers in the tables below link to original data sources. Scores link to the model pages that correspond to that specific fine-tuned model. These tables will be updated when more simple fine-tuned models are made available. ### Named Entity Recognition | Model | [CoNLL-2002](https://www.clips.uantwerpen.be/conll2002/ner/) | [SoNaR-1](https://ivdnt.org/downloads/taalmaterialen/tstc-sonar-corpus) | spaCy UD LassySmall | | ---------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | | **BERTje** | [**90.24**](https://huggingface.co/wietsedv/bert-base-dutch-cased-finetuned-conll2002-ner) | [**84.93**](https://huggingface.co/wietsedv/bert-base-dutch-cased-finetuned-sonar-ner) | [86.10](https://huggingface.co/wietsedv/bert-base-dutch-cased-finetuned-udlassy-ner) | | [mBERT](https://github.com/google-research/bert/blob/master/multilingual.md) | [88.61](https://huggingface.co/wietsedv/bert-base-multilingual-cased-finetuned-conll2002-ner) | [84.19](https://huggingface.co/wietsedv/bert-base-multilingual-cased-finetuned-sonar-ner) | [**86.77**](https://huggingface.co/wietsedv/bert-base-multilingual-cased-finetuned-udlassy-ner) | | [BERT-NL](http://textdata.nl) | 85.05 | 80.45 | 81.62 | | [RobBERT](https://github.com/iPieter/RobBERT) | 84.72 | 81.98 | 79.84 | ### Part-of-speech tagging | Model | [UDv2.5 LassySmall](https://universaldependencies.org/treebanks/nl_lassysmall/index.html) | | ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | | **BERTje** | **96.48** | | [mBERT](https://github.com/google-research/bert/blob/master/multilingual.md) | 96.20 | | [BERT-NL](http://textdata.nl) | 96.10 | | [RobBERT](https://github.com/iPieter/RobBERT) | 95.91 | ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ParulChaudhari) author: John Snow Labs name: distilbert_qa_parulchaudhari_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `ParulChaudhari`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_parulchaudhari_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768887661.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_parulchaudhari_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768887661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_parulchaudhari_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_parulchaudhari_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_parulchaudhari_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ParulChaudhari/distilbert-base-uncased-finetuned-squad --- layout: model title: Sentiment Analysis of German news author: John Snow Labs name: bert_sequence_classifier_news_sentiment date: 2022-01-18 tags: [german, sentiment, bert_sequence, de, open_source] task: Sentiment Analysis language: de edition: Spark NLP 3.3.4 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` ([link](https://huggingface.co/mdraw/german-news-sentiment-bert)) and it's been finetuned on news texts about migration for German language, leveraging `Bert` embeddings and `BertForSequenceClassification` for text classification purposes. ## Predicted Entities `positive`, `negative`, `neutral` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_news_sentiment_de_3.3.4_3.0_1642504435983.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_news_sentiment_de_3.3.4_3.0_1642504435983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['neutral'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_news_sentiment| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|de| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## Data Source [https://wortschatz.uni-leipzig.de/en/download/German](https://wortschatz.uni-leipzig.de/en/download/German) --- layout: model title: English asr_wav2vec2_large_tedlium TFWav2Vec2ForCTC from sanchit-gandhi author: John Snow Labs name: asr_wav2vec2_large_tedlium date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_tedlium` is a English model originally trained by sanchit-gandhi. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_tedlium_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_tedlium_en_4.2.0_3.0_1664094417234.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_tedlium_en_4.2.0_3.0_1664094417234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_tedlium", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_tedlium", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_tedlium| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ashhyun) author: John Snow Labs name: distilbert_qa_ashhyun_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `ashhyun`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ashhyun_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770017698.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ashhyun_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770017698.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ashhyun_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ashhyun_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ashhyun_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ashhyun/distilbert-base-uncased-finetuned-squad --- layout: model title: English image_classifier_vit_beer_vs_wine ViTForImageClassification from filipafcastro author: John Snow Labs name: image_classifier_vit_beer_vs_wine date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_beer_vs_wine` is a English model originally trained by filipafcastro. ## Predicted Entities `beer`, `wine` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_beer_vs_wine_en_4.1.0_3.0_1660166570749.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_beer_vs_wine_en_4.1.0_3.0_1660166570749.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_beer_vs_wine", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_beer_vs_wine", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_beer_vs_wine| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from Ndonga to English author: John Snow Labs name: opus_mt_ng_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ng, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ng` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ng_en_xx_2.7.0_2.4_1609169685419.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ng_en_xx_2.7.0_2.4_1609169685419.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ng_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Cased model (from OnsElleuch) author: John Snow Labs name: t5_logisgenerator date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `logisgenerator` is a English model originally trained by `OnsElleuch`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_logisgenerator_en_4.3.0_3.0_1675104908400.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_logisgenerator_en_4.3.0_3.0_1675104908400.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_logisgenerator","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_logisgenerator","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_logisgenerator| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|280.2 MB| ## References - https://huggingface.co/OnsElleuch/logisgenerator - https://pypi.org/project/keytotext/ - https://pepy.tech/project/keytotext - https://colab.research.google.com/github/gagan3012/keytotext/blob/master/notebooks/K2T.ipynb - https://share.streamlit.io/gagan3012/keytotext/UI/app.py - https://github.com/gagan3012/keytotext#api - https://hub.docker.com/r/gagan30/keytotext - https://keytotext.readthedocs.io/en/latest/?badge=latest - https://github.com/psf/black - https://socialify.git.ci/gagan3012/keytotext/image?description=1&forks=1&language=1&owner=1&stargazers=1&theme=Light --- layout: model title: Augment Company Names with NASDAQ database author: John Snow Labs name: finmapper_nasdaq_companyname date: 2022-08-09 tags: [en, finance, companies, tickers, nasdaq, data, augmentation, licensed] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model allows you to, given an extracted name of a company, get information about that company, including the Industry, the Sector and the Trading Symbol (ticker). It can be optionally combined with Entity Resolution to normalize first the name of the company. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FIN_LEG_COMPANY_AUGMENTATION/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_companyname_en_1.0.0_3.2_1660038424307.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_companyname_en_1.0.0_3.2_1660038424307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') tokenizer = nlp.Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained('finner_orgs_prods_alias', 'en', 'finance/models')\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") # Optional: To normalize the ORG name using NASDAQ data before the mapping ########################################################################## chunkToDoc = nlp.Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") chunk_embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use_lg", "en")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("chunk_embeddings") use_er_model = finance.SentenceEntityResolverModel.pretrained('finel_nasdaq_data_company_name', 'en', 'finance/models')\ .setInputCols("chunk_embeddings")\ .setOutputCol('normalized')\ .setDistanceFunction("EUCLIDEAN") ########################################################################## CM = finance.ChunkMapperModel()\ .pretrained('finmapper_nasdaq_companyname', 'en', 'finance/models')\ .setInputCols(["normalized"])\ #or ner_chunk without normalization .setOutputCol("mappings") pipeline = nlp.Pipeline().setStages([document_assembler, tokenizer, embeddings, ner_model, ner_converter, chunkToDoc, # Optional for normalization chunk_embeddings, # Optional for normalization use_er_model, # Optional for normalization CM]) text = """Altaba Inc. is a company which ...""" test_data = spark.createDataFrame([[text]]).toDF("text") model = pipeline.fit(test_data) lp = nlp.LightPipeline(model) lp.fullAnnotate(text) ```

## Results ```bash [Row(mappings=[Row(annotatorType='labeled_dependency', begin=0, end=10, result='AABA', metadata={'sentence': '0', 'chunk': '0', 'entity': 'Altaba Inc.', 'relation': 'ticker', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=10, result='Altaba Inc.', metadata={'sentence': '0', 'chunk': '0', 'entity': 'Altaba Inc.', 'relation': 'company_name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=10, result='Altaba', metadata={'sentence': '0', 'chunk': '0', 'entity': 'Altaba Inc.', 'relation': 'short_name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=10, result='Asset Management', metadata={'sentence': '0', 'chunk': '0', 'entity': 'Altaba Inc.', 'relation': 'industry', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=10, result='Financial Services', metadata={'sentence': '0', 'chunk': '0', 'entity': 'Altaba Inc.', 'relation': 'sector', 'all_relations': ''}, embeddings=[])])] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_nasdaq_companyname| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|210.5 KB| ## References https://data.world/johnsnowlabs/list-of-companies-in-nasdaq-exchanges --- layout: model title: Stopwords Remover for Greek (modern) language (663 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, el, open_source] task: Stop Words Removal language: el edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_el_3.4.1_3.0_1646672928398.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_el_3.4.1_3.0_1646672928398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+ |result | +-----------------+ |[καλύτεροι, μένα]| +-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|el| |Size:|3.8 KB| --- layout: model title: Translate Caucasian languages to English Pipeline author: John Snow Labs name: translate_cau_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, cau, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `cau` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_cau_en_xx_2.7.0_2.4_1609687516556.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_cau_en_xx_2.7.0_2.4_1609687516556.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_cau_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_cau_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.cau.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_cau_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for ICD10-CM (Augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_augmented date: 2023-05-31 tags: [licensed, en, clinical, entity_resolution, icd10cm] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD-10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it has been augmented with synonyms for making it more accurate. ## Predicted Entities `ICD-10-CM Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_4.4.2_3.0_1685503130827.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_4.4.2_3.0_1685503130827.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | ner_chunk| entity|icd10_code| resolutions| all_codes| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.4|[gestational diabetes mellitus [gestational diabetes mellitus], gestatio...| [O24.4, O24.41, O24.43, Z86.32, Z87.5, O24.31, O24.11, O24.1, O24.81]| |subsequent type two diabetes mellitus|PROBLEM| O24.11|[pre-existing type 2 diabetes mellitus [pre-existing type 2 diabetes mel...|[O24.11, E11.8, E11, E13.9, E11.9, E11.3, E11.44, Z86.3, Z86.39, E11.32,...| | obesity|PROBLEM| E66.9|[obesity [obesity, unspecified], abdominal obesity [other obesity], obes...|[E66.9, E66.8, Z68.41, Q13.0, E66, E66.01, Z86.39, E34.9, H35.50, Z83.49...| | a body mass index|PROBLEM| Z68.41|[finding of body mass index [body mass index [bmi] 40.0-44.9, adult], ob...|[Z68.41, E66.9, R22.9, Z68.1, R22.3, R22.1, Z68, R22.2, R22.0, R41.89, M...| | polyuria|PROBLEM| R35|[polyuria [polyuria], nocturnal polyuria [nocturnal polyuria], polyuric ...|[R35, R35.81, R35.8, E23.2, R31, R35.0, R82.99, N40.1, E72.3, O04.8, R30...| | polydipsia|PROBLEM| R63.1|[polydipsia [polydipsia], psychogenic polydipsia [other impulse disorder...|[R63.1, F63.89, E23.2, F63.9, O40, G47.5, M79.89, R63.2, R06.1, H53.8, I...| | poor appetite|PROBLEM| R63.0|[poor appetite [anorexia], poor feeding [feeding problem of newborn, uns...|[R63.0, P92.9, R43.8, R43.2, E86, R19.6, F52.0, Z72.4, R06.89, Z76.89, R...| | vomiting|PROBLEM| R11.1|[vomiting [vomiting], intermittent vomiting [nausea and vomiting], vomit...| [R11.1, R11, R11.10, G43.A1, P92.1, P92.09, G43.A, R11.13, R11.0]| | a respiratory tract infection|PROBLEM| J98.8|[respiratory tract infection [other specified respiratory disorders], up...|[J98.8, J06.9, A49.9, J22, J20.9, Z59.3, T17, J04.10, Z13.83, J18.9, P28...| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|1.4 GB| |Case sensitive:|false| --- layout: model title: Kannada RoBERTa Embeddings (from Chakita) author: John Snow Labs name: roberta_embeddings_KNUBert date: 2022-04-14 tags: [roberta, embeddings, kn, open_source] task: Embeddings language: kn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `KNUBert` is a Kannada model orginally trained by `Chakita`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_KNUBert_kn_3.4.2_3.0_1649948307214.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_KNUBert_kn_3.4.2_3.0_1649948307214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_KNUBert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|kn| |Size:|314.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/Chakita/KNUBert --- layout: model title: Relation extraction between Drugs and ADE (ReDL) author: John Snow Labs name: redl_ade_biobert date: 2023-01-14 tags: [relation_extraction, en, clinical, licensed, ade, biobert, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is an end-to-end trained BioBERT model, capable of Relating Drugs and adverse reactions caused by them; It predicts if an adverse event is caused by a drug or not. 1 : Shows the adverse event and drug entities are related, 0 : Shows the adverse event and drug entities are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/RE_ADE.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_ade_biobert_en_4.2.4_3.0_1673708531142.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_ade_biobert_en_4.2.4_3.0_1673708531142.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |-----------:|:----------|----------------:|--------------:|:----------|:----------|----------------:|--------------:|:---------------|-------------:| | 1 | DRUG | 12 | 18 | Lipitor | ADE | 52 | 65 | severe fatigue | 0.998156 | | 1 | DRUG | 97 | 105 | voltarene | ADE | 144 | 156 | muscle cramps | 0.985513 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_ade_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References This model is trained on custom data annotated by JSL. ## Benchmarking ```bash label Recall Precision F1 Support 0 0.829 0.895 0.861 1146 1 0.955 0.923 0.939 2454 Avg. 0.892 0.909 0.900 - Weighted-Avg. 0.915 0.914 0.914 - ``` --- layout: model title: English BertForQuestionAnswering model (from MrAnderson) author: John Snow Labs name: bert_qa_bert_base_512_full_trivia date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-512-full-trivia` is a English model orginally trained by `MrAnderson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_512_full_trivia_en_4.0.0_3.0_1654179670286.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_512_full_trivia_en_4.0.0_3.0_1654179670286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_512_full_trivia","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_512_full_trivia","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.trivia.bert.base_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_512_full_trivia| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MrAnderson/bert-base-512-full-trivia --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223765535.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223765535.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_only_classfn_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_only_classfn_epochs_1_shard_1_squad2.0 --- layout: model title: Extract Mentions of Response to Cancer Treatment author: John Snow Labs name: ner_oncology_response_to_treatment_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, ner, treatment] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to the patient"s response to the oncology treatment, including clinical response and changes in tumor size. Definitions of Predicted Entities: - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Size_Trend`: Terms related to the changes in the size of the tumor (such as "growth" or "reduced in size"). ## Predicted Entities `Line_Of_Therapy`, `Response_To_Treatment`, `Size_Trend` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_wip_en_4.0.0_3.0_1664585303681.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_wip_en_4.0.0_3.0_1664585303681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------|:----------------------| | first-line | Line_Of_Therapy | | recurrence | Response_To_Treatment | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_response_to_treatment_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|848.8 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Response_To_Treatment 233.0 81.0 120.0 353.0 0.74 0.66 0.70 Size_Trend 31.0 34.0 45.0 76.0 0.48 0.41 0.44 Line_Of_Therapy 82.0 11.0 5.0 87.0 0.88 0.94 0.91 macro_avg 346.0 126.0 170.0 516.0 0.70 0.67 0.68 micro_avg NaN NaN NaN NaN 0.73 0.67 0.70 ``` --- layout: model title: Recognize Entities OntoNotes pipeline - BERT Large author: John Snow Labs name: onto_recognize_entities_bert_large date: 2021-03-23 tags: [open_source, english, onto_recognize_entities_bert_large, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The onto_recognize_entities_bert_large is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_large_en_3.0.0_3.0_1616475201428.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_large_en_3.0.0_3.0_1616475201428.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('onto_recognize_entities_bert_large', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_large", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.ner.onto.bert.large').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:-------------------------------------------|:-------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[-0.262016534805297,.,...]] | ['O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'O'] | ['John Snow Labs'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_large| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_nh24 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-nh24` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nh24_en_4.3.0_3.0_1675113000788.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nh24_en_4.3.0_3.0_1675113000788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_nh24","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_nh24","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_nh24| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|636.9 MB| ## References - https://huggingface.co/google/t5-efficient-base-nh24 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English BertForQuestionAnswering model (from armageddon) author: John Snow Labs name: bert_qa_bert_base_uncased_squad2_covid_qa_deepset date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squad2-covid-qa-deepset` is a English model orginally trained by `armageddon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad2_covid_qa_deepset_en_4.0.0_3.0_1654181524986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad2_covid_qa_deepset_en_4.0.0_3.0_1654181524986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squad2_covid_qa_deepset","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squad2_covid_qa_deepset","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squad2_covid_qa_deepset| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/armageddon/bert-base-uncased-squad2-covid-qa-deepset --- layout: model title: Sentiment Analysis on texts about Airlines author: John Snow Labs name: distilbert_base_sequence_classifier_airlines date: 2022-02-18 tags: [airlines, distilbert, sequence_classification, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: DistilBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` ([link](https://huggingface.co/tasosk/distilbert-base-uncased-airlines)) and it's been trained on tasosk/airlines dataset, leveraging `Distil-BERT` embeddings and `DistilBertForSequenceClassification` for text classification purposes. The model classifies texts into two categories: `YES` for positive comments, and `NO` for negative. ## Predicted Entities `YES`, `NO` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_airlines_en_3.4.0_3.0_1645179643194.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_airlines_en_3.4.0_3.0_1645179643194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = DistilBertForSequenceClassification\ .pretrained('distilbert_base_sequence_classifier_airlines', 'en') \ .setInputCols(['token', 'document']) \ .setOutputCol('class') pipeline = Pipeline(stages=[document_assembler, tokenizer, sequenceClassifier]) example = spark.createDataFrame([["Jersey to London Gatwick with easyJet and another great flight. Due to the flight time, airport check-in was not open, however I'd checked in a few days before with the easyJet app which was very quick and convenient. Boarding was quick and we left a few minutes early, which is a bonus. The cabin crew were friendly and the aircraft was clean and comfortable. We arrived at Gatwick 5-10 minutes early, and disembarking was as quick as boarding. On the way back, we were about half an hour early landing, which was fantastic. For the short flight from JER-LGW, easyJet are ideal and a bit better than British Airways in my opinion, and the fares are just unmissable. Both flights for two adults cost £180. easyJet can expect my business in the near future."]]).toDF("text") result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val tokenClassifier = DistilBertForSequenceClassification.pretrained("distilbert_base_sequence_classifier_airlines", "en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val example = Seq("Jersey to London Gatwick with easyJet and another great flight. Due to the flight time, airport check-in was not open, however I'd checked in a few days before with the easyJet app which was very quick and convenient. Boarding was quick and we left a few minutes early, which is a bonus. The cabin crew were friendly and the aircraft was clean and comfortable. We arrived at Gatwick 5-10 minutes early, and disembarking was as quick as boarding. On the way back, we were about half an hour early landing, which was fantastic. For the short flight from JER-LGW, easyJet are ideal and a bit better than British Airways in my opinion, and the fares are just unmissable. Both flights for two adults cost £180. easyJet can expect my business in the near future.").toDF("text") val result = pipeline.fit(example).transform(example) ```

## Results ```bash ['YES'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_sequence_classifier_airlines| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[class]| |Language:|en| |Size:|249.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References [https://huggingface.co/datasets/tasosk/airlines](https://huggingface.co/datasets/tasosk/airlines) ## Benchmarking ```bash label score accuracy 0.9288 f1 0.9289 ``` --- layout: model title: Pipeline to Detect Chemicals in Medical Texts author: John Snow Labs name: bert_token_classifier_ner_chemicals_pipeline date: 2022-03-14 tags: [chemicals, bert_token_classifier, pipeline, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemicals](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemicals_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMICALS/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_3.0_1647256416720.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_3.0_1647256416720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python chemicals_pipeline = PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") chemicals_pipeline.annotate("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ``` ```scala val chemicals_pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") chemicals_pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemicals_pipeline").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |p - choloroaniline |CHEM | |chlorhexidine - digluconate|CHEM | |kanamycin |CHEM | |colistin |CHEM | |povidone - iodine |CHEM | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.3 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForTokenClassifier - NerConverter - Finisher --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from botika) author: John Snow Labs name: distilbert_qa_botika_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `botika`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_botika_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770249638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_botika_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770249638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_botika_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_botika_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_botika_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/botika/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from graviraja) author: John Snow Labs name: distilbert_qa_graviraja_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `graviraja`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_graviraja_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770884217.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_graviraja_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770884217.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_graviraja_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_graviraja_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_graviraja_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/graviraja/distilbert-base-uncased-finetuned-squad --- layout: model title: Extract Demographic Entities from Voice of the Patient Documents (embeddings_clinical_large) author: John Snow Labs name: ner_vop_demographic_emb_clinical_large date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, demographic] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts demographic terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `Gender`, `Employment`, `RaceEthnicity`, `Age`, `Substance`, `RelationshipStatus`, `SubstanceQuantity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_demographic_emb_clinical_large_en_4.4.3_3.0_1686075195884.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_demographic_emb_clinical_large_en_4.4.3_3.0_1686075195884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------|:--------------| | grandma | Gender | | who's 85 | Age | | Black | RaceEthnicity | | doctors | Employment | | her | Gender | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_demographic_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_large| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Gender 1298 21 19 1317 0.98 0.99 0.98 Employment 1180 50 63 1243 0.96 0.95 0.95 RaceEthnicity 31 2 2 33 0.94 0.94 0.94 Age 549 45 33 582 0.92 0.94 0.93 Substance 391 56 30 421 0.87 0.93 0.90 RelationshipStatus 18 3 6 24 0.86 0.75 0.80 SubstanceQuantity 61 14 24 85 0.81 0.72 0.76 macro_avg 3528 191 177 3705 0.91 0.89 0.89 micro_avg 3528 191 177 3705 0.95 0.95 0.95 ``` --- layout: model title: Sentence Entity Resolver for Snomed Concepts, CT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_findings language: en nav_key: models repository: clinical/models date: 2020-11-27 task: Entity Resolution edition: Healthcare NLP 2.6.4 spark_version: 2.4 tags: [clinical,entity_resolution,en] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model maps extracted medical entities to Snomed codes (CT version) using chunk embeddings. {:.h2_title} ## Predicted Entities Snomed Codes and their normalized definition with ``sbiobert_base_cased_mli`` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_en_2.6.4_2.4_1606235762315.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_en_2.6.4_2.4_1606235762315.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use ```sbiobertresolve_snomed_findings``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. No need to set ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_snomed_findings","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_findings","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_resolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 38341003| 0.3234|hypertension:::hy...|38341003:::155295...| |chronic renal ins...| 83|109| PROBLEM|723190009| 0.7522|chronic renal ins...|723190009:::70904...| | COPD| 113|116| PROBLEM| 13645005| 0.1226|copd - chronic ob...|13645005:::155565...| | gastritis| 120|128| PROBLEM|235653009| 0.2444|gastritis:::gastr...|235653009:::45560...| | TIA| 136|138| PROBLEM|275382005| 0.0766|cerebral trauma (...|275382005:::44739...| |a non-ST elevatio...| 182|202| PROBLEM|233843008| 0.2224|silent myocardial...|233843008:::19479...| |Guaiac positive s...| 208|229| PROBLEM| 59614000| 0.9678|guaiac-positive s...|59614000:::703960...| |cardiac catheteri...| 295|317| TEST|301095005| 0.2584|cardiac finding::...|301095005:::25090...| | PTCA| 324|327|TREATMENT|373108000| 0.0809|post percutaneous...|373108000:::25103...| | mid LAD lesion| 332|345| PROBLEM|449567000| 0.0900|overriding left v...|449567000:::46140...| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_snomed_findings | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.4 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on SNOMED (CT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: Fast Neural Machine Translation Model from Swedish to English author: John Snow Labs name: opus_mt_sv_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, sv, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `sv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_sv_en_xx_2.7.0_2.4_1609170150107.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_sv_en_xx_2.7.0_2.4_1609170150107.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_sv_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-wwm-squadv2-x2.15-f83.2-d25-hybrid-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1_en_4.0.0_3.0_1654537627313.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1_en_4.0.0_3.0_1654537627313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_uncased_v2_x2.15_f83.2_d25_hybrid.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_wwm_squadv2_x2.15_f83.2_d25_hybrid_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|455.4 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-large-uncased-wwm-squadv2-x2.15-f83.2-d25-hybrid-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: Detect PHI for Deidentification purposes (Spanish, reduced entities, augmented data) author: John Snow Labs name: ner_deid_generic_augmented date: 2022-02-16 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 8 entities (1 more than the `ner_deid_generic` ner model). This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf dataset, several data augmentation mechanisms and has been augmented with MEDDOCAN Spanish Deidentification corpus (compared to `ner_deid_generic` which does not include it). It's a generalized version of `ner_deid_subentity_augmented`. ## Predicted Entities `CONTACT`, `NAME`, `DATE`, `ID`, `LOCATION`, `PROFESSION`, `AGE`, `SEX` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_es_3.3.4_3.0_1645006125653.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_es_3.3.4_3.0_1645006125653.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("word_embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_generic_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] df = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(df).transform(df) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models") .setInputCols(Array("sentence","token")) .setOutputCol("word_embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_generic_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","word_embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val df = Seq(text).toDF("text") val results = pipeline.fit(df).transform(df) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.generic_augmented").predict(""" Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-NAME| | Miguel| I-NAME| | Martínez| I-NAME| | ,| O| | un| B-SEX| | varón| I-SEX| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-LOCATION| | ,| O| | España| B-LOCATION| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19|B-PROFESSION| | el| O| | dia| O| | 14| O| | de| O| | Marzo| O| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| B-LOCATION| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-LOCATION| | San| I-LOCATION| | Carlos| I-LOCATION| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|15.0 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 CONTACT 185.0 3.0 0.0 185.0 0.984 1.0 0.992 NAME 2066.0 138.0 106.0 2172.0 0.9374 0.9512 0.9442 DATE 1017.0 18.0 18.0 1035.0 0.9826 0.9826 0.9826 ORGANIZATION 2468.0 482.0 332.0 2800.0 0.8366 0.8814 0.8584 ID 65.0 5.0 3.0 68.0 0.9286 0.9559 0.942 SEX 678.0 8.0 15.0 693.0 0.9883 0.9784 0.9833 LOCATION 2532.0 358.0 420.0 2952.0 0.8761 0.8577 0.8668 PROFESSION 246.0 9.0 31.0 277.0 0.9647 0.8881 0.9248 AGE 547.0 8.0 9.0 556.0 0.9856 0.9838 0.9847 macro - - - - - - 0.9421 micro - - - - - - 0.9092 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from aszidon) Custom Version-3 author: John Snow Labs name: distilbert_qa_custom3 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom3` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom3_en_4.0.0_3.0_1654727988638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom3_en_4.0.0_3.0_1654727988638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.custom3.by_aszidon").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom3 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from claytonsamples) author: John Snow Labs name: xlmroberta_ner_claytonsamples_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `claytonsamples`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_claytonsamples_base_finetuned_panx_de_4.1.0_3.0_1660431662923.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_claytonsamples_base_finetuned_panx_de_4.1.0_3.0_1660431662923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_claytonsamples_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_claytonsamples_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_claytonsamples_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/claytonsamples/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Thai BertForQuestionAnswering model (from airesearch) author: John Snow Labs name: bert_qa_bert_base_multilingual_cased_finetune_qa date: 2022-06-02 tags: [th, open_source, question_answering, bert] task: Question Answering language: th edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetune-qa` is a Thai model orginally trained by `airesearch`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetune_qa_th_4.0.0_3.0_1654179974020.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetune_qa_th_4.0.0_3.0_1654179974020.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_cased_finetune_qa","th") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_cased_finetune_qa","th") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("th.answer_question.bert.multilingual_base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_cased_finetune_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|th| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/airesearch/bert-base-multilingual-cased-finetune-qa - https://github.com/vistec-AI/thai2transformers/blob/dev/scripts/downstream/train_question_answering_lm_finetuning.py - https://wandb.ai/cstorm125/wangchanberta-qa --- layout: model title: Stopwords Remover for French language (507 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, fr, open_source] task: Stop Words Removal language: fr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_fr_3.4.1_3.0_1646673106300.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_fr_3.4.1_3.0_1646673106300.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------+ |result | +-------------+ |[n'es, mieux]| +-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|fr| |Size:|2.9 KB| --- layout: model title: ALBERT Embeddings (XLarge Uncase) author: John Snow Labs name: albert_xlarge_uncased date: 2020-04-28 task: Embeddings language: en nav_key: models edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ALBERT is "A Lite" version of BERT, a popular unsupervised language representation learning algorithm. ALBERT uses parameter-reduction techniques that allow for large-scale configurations, overcome previous memory limitations, and achieve better behavior with respect to model degradation. The details are described in the paper "[ALBERT: A Lite BERT for Self-supervised Learning of Language Representations.](https://arxiv.org/abs/1909.11942)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_xlarge_uncased_en_2.5.0_2.4_1588073443653.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_xlarge_uncased_en_2.5.0_2.4_1588073443653.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = AlbertEmbeddings.pretrained("albert_xlarge_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = AlbertEmbeddings.pretrained("albert_xlarge_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.albert.xlarge_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_albert_xlarge_uncased_embeddings I [-0.4735468626022339, -0.03991951420903206, -1... love [-0.4254034459590912, -0.371383935213089, -0.3... NLP [0.7200506329536438, -0.12543179094791412, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_xlarge_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|2048| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/albert_xlarge/3](https://tfhub.dev/google/albert_xlarge/3) --- layout: model title: Polish T5ForConditionalGeneration Base Cased model (from azwierzc) author: John Snow Labs name: t5_plt5_base_poquad date: 2023-01-30 tags: [pl, open_source, t5, tensorflow] task: Text Generation language: pl edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `plt5-base-poquad` is a Polish model originally trained by `azwierzc`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_plt5_base_poquad_pl_4.3.0_3.0_1675106743524.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_plt5_base_poquad_pl_4.3.0_3.0_1675106743524.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_plt5_base_poquad","pl") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_plt5_base_poquad","pl") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_plt5_base_poquad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|pl| |Size:|1.1 GB| ## References - https://huggingface.co/azwierzc/plt5-base-poquad --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-28 tags: [cs, licensed, legal, ner, mapa] task: Named Entity Recognition language: cs edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Czech` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_cs_1.0.0_3.0_1682668776380.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_cs_1.0.0_3.0_1682668776380.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_czech_legal","cs")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "cs", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""V roce 2007 uzavřela společnost Alpenrind, dříve S GmbH, se společností Martin-Meat usazenou v Maďarsku smlouvu, podle níž se posledně uvedená společnost zavázala k porcování masa a jeho balení v rozsahu 25 půlek jatečně upravených těl skotu týdně."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-----------+------------+ |chunk |ner_label | +-----------+------------+ |2007 |DATE | |Alpenrind |ORGANISATION| |Martin-Meat|ORGANISATION| |Maďarsku |ADDRESS | |25 půlek |AMOUNT | +-----------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|cs| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.80 0.67 0.73 36 AMOUNT 1.00 1.00 1.00 5 DATE 0.98 0.98 0.98 56 ORGANISATION 0.64 0.66 0.65 32 PERSON 0.75 0.82 0.78 66 micro-avg 0.81 0.82 0.81 195 macro-avg 0.83 0.82 0.83 195 weighted-avg 0.81 0.82 0.81 195 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from shmuelamar) author: John Snow Labs name: roberta_qa_re date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `REQA-RoBERTa` is a English model originally trained by `shmuelamar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_re_en_4.3.0_3.0_1674208450623.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_re_en_4.3.0_3.0_1674208450623.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_re","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_re","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_re| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/shmuelamar/REQA-RoBERTa --- layout: model title: Detect Clinical Entities in Romanian (w2v_cc_300d) author: John Snow Labs name: ner_clinical date: 2022-07-01 tags: [licenced, clinical, ro, ner, w2v, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract clinical entities from Romanian clinical texts. This model is trained using Romanian `w2v_cc_300d` embeddings. ## Predicted Entities `Measurements`, `Form`, `Symptom`, `Route`, `Procedure`, `Disease_Syndrome_Disorder`, `Score`, `Drug_Ingredient`, `Pulse`, `Frequency`, `Date`, `Body_Part`, `Drug_Brand_Name`, `Time`, `Direction`, `Dosage`, `Medical_Device`, `Imaging_Technique`, `Test`, `Imaging_Findings`, `Imaging_Test`, `Test_Result`, `Weight`, `Clinical_Dept`, `Units` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_ro_4.0.0_3.0_1656687302322.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_ro_4.0.0_3.0_1656687302322.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "ro") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "ro", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) sample_text = """ Solicitare: Angio CT cardio-toracic Dg. de trimitere Atrezie de valva pulmonara. Hipoplazie VS. Atrezie VAV stang. Anastomoza Glenn. Sp. Tromboza la nivelul anastomozei. Trimis de: Sectia Clinica Cardiologie (dr. Sue T.) Procedura Aparat GE Revolution HD. Branula albastra montata la nivelul membrului superior drept. Se administreaza 30 ml Iomeron 350 cu flux 2.2 ml/s, urmate de 20 ml ser fiziologic cu acelasi flux. Se efectueaza o examinare angio-CT cardiotoracica cu achizitii secventiale prospective la o frecventa cardiaca medie de 100/min.""" data = spark.createDataFrame([[sample_text]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","ro") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical", "ro","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter)) val data = Seq("""Solicitare: Angio CT cardio-toracic Dg. de trimitere Atrezie de valva pulmonara. Hipoplazie VS. Atrezie VAV stang. Anastomoza Glenn. Sp. Tromboza la nivelul anastomozei. Trimis de: Sectia Clinica Cardiologie (dr. Sue T.) Procedura Aparat GE Revolution HD. Branula albastra montata la nivelul membrului superior drept. Se administreaza 30 ml Iomeron 350 cu flux 2.2 ml/s, urmate de 20 ml ser fiziologic cu acelasi flux. Se efectueaza o examinare angio-CT cardiotoracica cu achizitii secventiale prospective la o frecventa cardiaca medie de 100/min.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ro.med_ner.clinical").predict(""" Solicitare: Angio CT cardio-toracic Dg. de trimitere Atrezie de valva pulmonara. Hipoplazie VS. Atrezie VAV stang. Anastomoza Glenn. Sp. Tromboza la nivelul anastomozei. Trimis de: Sectia Clinica Cardiologie (dr. Sue T.) Procedura Aparat GE Revolution HD. Branula albastra montata la nivelul membrului superior drept. Se administreaza 30 ml Iomeron 350 cu flux 2.2 ml/s, urmate de 20 ml ser fiziologic cu acelasi flux. Se efectueaza o examinare angio-CT cardiotoracica cu achizitii secventiale prospective la o frecventa cardiaca medie de 100/min.""") ```

## Results ```bash +--------------------------+-------------------------+ |chunks |entities | +--------------------------+-------------------------+ |Angio CT |Imaging_Test | |cardio-toracic |Body_Part | |Atrezie |Disease_Syndrome_Disorder| |valva pulmonara |Body_Part | |Hipoplazie |Disease_Syndrome_Disorder| |VS |Body_Part | |Atrezie |Disease_Syndrome_Disorder| |VAV stang |Body_Part | |Anastomoza Glenn |Disease_Syndrome_Disorder| |Sp |Body_Part | |Tromboza |Disease_Syndrome_Disorder| |Sectia Clinica Cardiologie|Clinical_Dept | |GE Revolution HD |Medical_Device | |Branula albastra |Medical_Device | |membrului superior |Body_Part | |drept |Direction | |30 ml |Dosage | |Iomeron 350 |Drug_Ingredient | |2.2 ml/s |Dosage | |20 ml |Dosage | +--------------------------+-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|15.0 MB| ## Benchmarking ```bash label precision recall f1-score support Body_Part 0.87 0.90 0.88 689 Clinical_Dept 0.68 0.62 0.65 97 Date 1.00 0.99 0.99 87 Direction 0.64 0.74 0.69 50 Disease_Syndrome_Disorder 0.69 0.66 0.67 123 Dosage 0.74 0.97 0.84 38 Drug_Ingredient 0.98 0.92 0.95 48 Form 1.00 1.00 1.00 6 Imaging_Findings 0.74 0.76 0.75 202 Imaging_Technique 0.92 0.88 0.90 26 Imaging_Test 0.93 0.97 0.95 208 Measurements 0.70 0.67 0.69 214 Medical_Device 0.92 0.81 0.86 42 Pulse 0.82 1.00 0.90 9 Route 0.97 0.91 0.94 33 Score 0.91 0.95 0.93 41 Time 1.00 1.00 1.00 28 Units 0.60 0.89 0.71 88 Weight 1.00 1.00 1.00 9 micro-avg 0.82 0.84 0.83 2054 macro-avg 0.70 0.72 0.71 2054 weighted-avg 0.81 0.84 0.82 2054 ``` --- layout: model title: Part of Speech for Norwegian author: John Snow Labs name: pos_ud_bokmaal date: 2022-01-11 tags: [pos, norwegian, nb, open_source] task: Part of Speech Tagging language: nb edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. This model was trained using the dataset available at https://universaldependencies.org {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bokmaal_nb_3.4.0_3.0_1641902661339.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bokmaal_nb_3.4.0_3.0_1641902661339.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pos = PerceptronModel.pretrained("pos_ud_bokmaal", "nb") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.") ``` ```scala val pos = PerceptronModel.pretrained("pos_ud_bokmaal", "nb") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene."""] pos_df = nlu.load('nb.pos.ud_bokmaal').predict(text) pos_df ```

## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='DET', metadata={'word': 'Annet'}), Row(annotatorType='pos', begin=6, end=8, result='SCONJ', metadata={'word': 'enn'}), Row(annotatorType='pos', begin=10, end=10, result='PART', metadata={'word': 'å'}), Row(annotatorType='pos', begin=12, end=15, result='AUX', metadata={'word': 'være'}), Row(annotatorType='pos', begin=17, end=22, result='NOUN', metadata={'word': 'kongen'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bokmaal| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Language:|nb| |Size:|17.7 KB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel --- layout: model title: French CamemBert Embeddings (from ppletscher) author: John Snow Labs name: camembert_embeddings_dummy date: 2022-05-23 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy` is a French model orginally trained by `ppletscher`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_dummy_fr_3.4.4_3.0_1653321214351.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_dummy_fr_3.4.4_3.0_1653321214351.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_dummy| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/ppletscher/dummy --- layout: model title: SDOH Under Treatment For Classification author: John Snow Labs name: genericclassifier_sdoh_under_treatment_sbiobert_cased_mli date: 2023-04-27 tags: [en, licensed, clinical, sdoh, generic_classifier, under_treatment, biobert] task: Text Classification language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting if the patient is under treatment or not. If under treatment is not mentioned in the text, it is regarded as “not under treatment”. The model is trained by using GenericClassifierApproach annotator. `Under_Treatment`: The patient is under treatment. `Not_Under_Treatment_Or_Not_Mentioned`: The patient is not under treatment or it is not mentioned in the clinical notes. ## Predicted Entities `Under_Treatment`, `Not_Under_Treatment_Or_Not_Mentioned` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_under_treatment_sbiobert_cased_mli_en_4.3.2_3.0_1682608513576.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_under_treatment_sbiobert_cased_mli_en_4.3.2_3.0_1682608513576.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+--------------------------------------+ | text| result| +----------------------------------------------------------------------------------------------------+--------------------------------------+ |Sarah, a 55-year-old woman with a history of high cholesterol and a family history of heart disea...| [Under_Treatment]| |John, a 60-year-old man with a history of smoking and high blood pressure, presented to his prima...|[Not_Under_Treatment_Or_Not_Mentioned]| +----------------------------------------------------------------------------------------------------+--------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_under_treatment_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH Project ## Benchmarking ```bash label precision recall f1-score support Not_Under_Treatment_Or_Not_Mentioned 0.86 0.68 0.76 222 Under_Treatment 0.86 0.94 0.90 450 accuracy - - 0.86 672 macro-avg 0.86 0.81 0.83 672 weighted-avg 0.86 0.86 0.85 672 ``` --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-wwm-squadv2-x2.63-f82.6-d16-hybrid-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1_en_4.0.0_3.0_1654183687318.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1_en_4.0.0_3.0_1654183687318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_uncased_v2_x2.63_f82.6_d16_hybrid.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_wwm_squadv2_x2.63_f82.6_d16_hybrid_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|349.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-large-uncased-wwm-squadv2-x2.63-f82.6-d16-hybrid-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: German XlmRoBertaForQuestionAnswering (from bhavikardeshna) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_german date: 2022-06-23 tags: [de, open_source, question_answering, xlmroberta] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-german` is a German model originally trained by `bhavikardeshna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_german_de_4.0.0_3.0_1655989699247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_german_de_4.0.0_3.0_1655989699247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_german","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_german","de") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_german| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|de| |Size:|883.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bhavikardeshna/xlm-roberta-base-german --- layout: model title: English BertForSequenceClassification Tiny Cased model (from mrm8488) author: John Snow Labs name: bert_sequence_classifier_tiny_finetuned_fake_news_detection date: 2022-07-13 tags: [en, open_source, bert, sequence_classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-finetuned-fake-news-detection` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_tiny_finetuned_fake_news_detection_en_4.0.0_3.0_1657720809070.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_tiny_finetuned_fake_news_detection_en_4.0.0_3.0_1657720809070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = BertForSequenceClassification.pretrained("bert_sequence_classifier_tiny_finetuned_fake_news_detection","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = BertForSequenceClassification.pretrained("bert_sequence_classifier_tiny_finetuned_fake_news_detection","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_tiny_finetuned_fake_news_detection| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|16.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/bert-tiny-finetuned-fake-news-detection --- layout: model title: Translate South Slavic languages to English Pipeline author: John Snow Labs name: translate_zls_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, zls, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `zls` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_zls_en_xx_2.7.0_2.4_1609688390022.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_zls_en_xx_2.7.0_2.4_1609688390022.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_zls_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_zls_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.zls.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_zls_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Assertion Status from Smoking Status Entity author: John Snow Labs name: assertion_oncology_smoking_status_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of the Smoking_Status entity. It classifies extractions as Present, Past or Absent. ## Predicted Entities `Absent`, `Past`, `Present` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_smoking_status_wip_en_4.1.0_3.0_1664641973214.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_smoking_status_wip_en_4.1.0_3.0_1664641973214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------|:---------------|:------------| | smoking | Smoking_Status | Past | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_smoking_status_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Absent 0.75 1.00 0.86 12.0 Past 0.78 0.93 0.85 15.0 Present 1.00 0.46 0.63 13.0 macro-avg 0.84 0.80 0.78 40.0 weighted-avg 0.84 0.80 0.78 40.0 ``` --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_scrambled_squad_5 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-5` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_5_en_4.0.0_3.0_1655734166418.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_5_en_4.0.0_3.0_1655734166418.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_5","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_5","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_scrambled_5.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_scrambled_squad_5| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-5 --- layout: model title: Legal Legends Clause Binary Classifier author: John Snow Labs name: legclf_legends_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `legends` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `legends` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_legends_clause_en_1.0.0_3.2_1660123668543.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_legends_clause_en_1.0.0_3.2_1660123668543.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_legends_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[legends]| |[other]| |[other]| |[legends]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_legends_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support legends 0.98 0.98 0.98 57 other 0.99 0.99 0.99 128 accuracy - - 0.99 185 macro-avg 0.99 0.99 0.99 185 weighted-avg 0.99 0.99 0.99 185 ``` --- layout: model title: German asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377 date: 2022-09-26 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377_de_4.2.0_3.0_1664189496988.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377_de_4.2.0_3.0_1664189496988.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s377| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Fast Neural Machine Translation Model from Celtic Languages to English author: John Snow Labs name: opus_mt_cel_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, cel, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `cel` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_cel_en_xx_2.7.0_2.4_1609168916224.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_cel_en_xx_2.7.0_2.4_1609168916224.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_cel_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_de_4.2.0_3.0_1664112235753.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_de_4.2.0_3.0_1664112235753.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Sentence Embeddings - sbert medium (tuned) author: John Snow Labs name: sbert_jsl_medium_uncased date: 2021-05-14 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_uncased_en_3.0.3_2.4_1621017111185.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_medium_uncased_en_3.0.3_2.4_1621017111185.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_uncased","en","clinical/models")\ .setInputCols(["sentence"])\ .setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_medium_uncased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_medium_uncased| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|false| ## Data Source Tuned on MedNLI dataset ## Benchmarking ```bash MedNLI Score Acc 0.724 STS(cos) 0.743 ``` --- layout: model title: Translate English to Tongan Pipeline author: John Snow Labs name: translate_en_to date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, to, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `to` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_to_xx_2.7.0_2.4_1609686800591.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_to_xx_2.7.0_2.4_1609686800591.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_to", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_to", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.to').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_to| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from rahulchakwate) author: John Snow Labs name: roberta_qa_roberta_large_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-finetuned-squad` is a English model originally trained by `rahulchakwate`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_finetuned_squad_en_4.0.0_3.0_1655736911099.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_finetuned_squad_en_4.0.0_3.0_1655736911099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.large.by_rahulchakwate").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rahulchakwate/roberta-large-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6_en_4.3.0_3.0_1674213584769.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6_en_4.3.0_3.0_1674213584769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|439.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-6 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8_en_4.3.0_3.0_1674214068825.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8_en_4.3.0_3.0_1674214068825.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_128_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|423.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-8 --- layout: model title: English image_classifier_vit_dog ViTForImageClassification from Sena author: John Snow Labs name: image_classifier_vit_dog date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_dog` is a English model originally trained by Sena. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_en_4.1.0_3.0_1660169568437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_en_4.1.0_3.0_1660169568437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_dog", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_dog", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_dog| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect Adverse Drug Events (MedicalBertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ade_tweet_binary_pipeline date: 2023-03-20 tags: [clinical, licensed, ade, en, medicalbertfortokenclassification, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ade_tweet_binary](https://nlp.johnsnowlabs.com/2022/07/29/bert_token_classifier_ade_tweet_binary_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ade_tweet_binary_pipeline_en_4.3.0_3.2_1679298990358.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ade_tweet_binary_pipeline_en_4.3.0_3.2_1679298990358.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ade_tweet_binary_pipeline", "en", "clinical/models") text = '''I used to be on paxil but that made me more depressed and prozac made me angry. Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ade_tweet_binary_pipeline", "en", "clinical/models") val text = "I used to be on paxil but that made me more depressed and prozac made me angry. Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:------------|-------------:| | 0 | depressed | 44 | 52 | ADE | 0.999755 | | 1 | angry | 73 | 77 | ADE | 0.999608 | | 2 | insulin blocking | 97 | 112 | ADE | 0.738712 | | 3 | sugar crashes | 147 | 159 | ADE | 0.993742 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ade_tweet_binary_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Multilingual XLMRoBerta Embeddings (from castorini) author: John Snow Labs name: xlmroberta_embeddings_afriberta_large date: 2022-05-13 tags: [ha, yo, ig, am, so, open_source, xlm_roberta, embeddings, xx, afriberta] task: Embeddings language: xx edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `afriberta_large` is a Multilingual model orginally trained by `castorini`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_afriberta_large_xx_3.4.4_3.0_1652439242600.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_afriberta_large_xx_3.4.4_3.0_1652439242600.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_afriberta_large| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|471.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/castorini/afriberta_large - https://github.com/keleog/afriberta --- layout: model title: Detect Chemical Compounds and Genes author: John Snow Labs name: ner_chemprot_clinical date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a pre-trained model that can be used to automatically detect all chemical compounds and gene mentions from medical texts. ## Predicted Entities : `CHEMICAL`, `GENE-Y`, `GENE-N` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMPROT_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CHEMPROT_CLINICAL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_clinical_en_3.0.0_3.0_1617208430062.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_clinical_en_3.0.0_3.0_1617208430062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_chemprot_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium."]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_chemprot_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemprot.clinical").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

## Results ```bash +----+---------------------------------+---------+-------+----------+ | | chunk | begin | end | entity | +====+=================================+=========+=======+==========+ | 0 | Keratinocyte growth factor | 0 | 25 | GENE-Y | +----+---------------------------------+---------+-------+----------+ | 1 | acidic fibroblast growth factor | 31 | 61 | GENE-Y | +----+---------------------------------+---------+-------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemprot_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source This model was trained on the ChemProt corpus using 'embeddings_clinical' embeddings. Make sure you use the same embeddings when running the model. ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|:--------------|-------:|------:|-----:|---------:|---------:|---------:| | 0 | B-GENE-Y | 4650 | 1090 | 838 | 0.810105 | 0.847303 | 0.828286 | | 1 | B-GENE-N | 1732 | 981 | 1019 | 0.638408 | 0.629589 | 0.633968 | | 2 | I-GENE-Y | 1846 | 571 | 573 | 0.763757 | 0.763125 | 0.763441 | | 3 | B-CHEMICAL | 7512 | 804 | 1136 | 0.903319 | 0.86864 | 0.88564 | | 4 | I-CHEMICAL | 1059 | 169 | 253 | 0.862378 | 0.807165 | 0.833858 | | 5 | I-GENE-N | 1393 | 853 | 598 | 0.620214 | 0.699648 | 0.657541 | | 6 | Macro-average | 18192 | 4468 | 4417 | 0.766363 | 0.769245 | 0.767801 | | 7 | Micro-average | 18192 | 4468 | 4417 | 0.802824 | 0.804635 | 0.803729 | ``` --- layout: model title: Arabic BertForMaskedLM Large Cased model (from aubmindlab) author: John Snow Labs name: bert_embeddings_large_arabertv02 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-arabertv02` is a Arabic model originally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabertv02_ar_4.2.4_3.0_1670019689670.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabertv02_ar_4.2.4_3.0_1670019689670.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabertv02","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabertv02","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_arabertv02| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|1.4 GB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-large-arabertv02 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-large-arabertv02/blob/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: Map Companies to their Acquisitions (wikipedia, en) author: John Snow Labs name: finmapper_wikipedia_parentcompanies date: 2023-01-13 tags: [parent, companies, subsidiaries, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This models allows you to, given an extracter ORG, retrieve all the parent / subsidiary /companies acquired and/or in the same group than it. IMPORTANT: This requires an exact match as the name appears in Wikidata. If you are not sure the name is the same, pleas run `finmapper_wikipedia_parentcompanies` to normalize the company name first. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_wikipedia_parentcompanies_en_1.0.0_3.0_1673610612510.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_wikipedia_parentcompanies_en_1.0.0_3.0_1673610612510.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained('finner_orgs_prods_alias', 'en', 'finance/models')\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") # Optional: To normalize the ORG name using Wikipedia data before the mapping ########################################################################## chunkToDoc = nlp.Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") chunk_embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk_doc") \ .setOutputCol("sentence_embeddings") use_er_model = finance.SentenceEntityResolverModel.pretrained("finel_wikipedia_parentcompanies", "en", "finance/models") \ .setInputCols(["ner_chunk_doc", "sentence_embeddings"]) \ .setOutputCol("normalized")\ .setDistanceFunction("EUCLIDEAN") ########################################################################## cm = finance.ChunkMapperModel()\ .pretrained("finmapper_wikipedia_parentcompanies", "en", "finance/models")\ .setInputCols(["normalized"])\ .setOutputCol("mappings") # or ner_chunk for non normalized versions nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, ner_model, ner_converter, chunkToDoc, chunk_embeddings, use_er_model, cm ]) text = ["""Barclays is an American multinational bank which operates worldwide."""] test_data = spark.createDataFrame([text]).toDF("text") model = nlpPipeline.fit(test_data) lp = nlp.LightPipeline(model) lp.annotate(text) ```

## Results ```bash {'mappings': ['http://www.wikidata.org/entity/Q245343', 'Barclays@en-ca', 'http://www.wikidata.org/prop/direct/P355', 'is_parent_of', 'London Stock Exchange@en', 'BARC', 'בנק ברקליס@he', 'http://www.wikidata.org/entity/Q29488227'], ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_wikipedia_parentcompanies| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|852.6 KB| ## References Wikidata --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_greek_1 TFWav2Vec2ForCTC from skylord author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_greek_1 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_greek_1` is a Modern Greek (1453-) model originally trained by skylord. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_greek_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_greek_1_el_4.2.0_3.0_1664110254247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_greek_1_el_4.2.0_3.0_1664110254247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_greek_1', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_greek_1", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_greek_1| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: NER Pipeline - Voice of the Patient author: John Snow Labs name: ner_vop_pipeline date: 2023-06-09 tags: [pipeline, ner, en, licensed, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes the full taxonomy Named-Entity Recognition model to extract information from health-related text in colloquial language. This pipeline extracts diagnoses, treatments, tests, anatomical references and demographic entities. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_pipeline_en_4.4.3_3.0_1686338017684.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_pipeline_en_4.4.3_3.0_1686338017684.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_vop_pipeline", "en", "clinical/models") pipeline.annotate(" Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_vop_pipeline", "en", "clinical/models") val result = pipeline.annotate(" Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ") ```

## Results ```bash | chunk | ner_label | |:---------------------|:-----------------------| | 20 year old | Age | | girl | Gender | | hyperthyroid | Disease | | 1 month ago | DateTime | | weak | Symptom | | light | Symptom | | panic attacks | PsychologicalCondition | | depression | PsychologicalCondition | | left | Laterality | | chest | BodyPart | | pain | Symptom | | increased | TestResult | | heart rate | VitalTest | | rapidly | Modifier | | weight loss | Symptom | | 4 months | Duration | | hospital | ClinicalDept | | discharged | AdmissionDischarge | | hospital | ClinicalDept | | blood tests | Test | | brain | BodyPart | | mri | Test | | ultrasound scan | Test | | endoscopy | Procedure | | doctors | Employment | | homeopathy doctor | Employment | | he | Gender | | hyperthyroid | Disease | | TSH | Test | | 0.15 | TestResult | | T3 | Test | | T4 | Test | | normal | TestResult | | b12 deficiency | Disease | | vitamin D deficiency | Disease | | weekly | Frequency | | supplement | Drug | | vitamin D | Drug | | 1000 mcg | Dosage | | b12 | Drug | | daily | Frequency | | homeopathy medicine | Drug | | 40 days | Duration | | after 30 days | DateTime | | TSH | Test | | 0.5 | TestResult | | now | DateTime | | weakness | Symptom | | depression | PsychologicalCondition | | last week | DateTime | | rapid | TestResult | | heartrate | VitalTest | | homeopathy | Treatment | | thyroid | BodyPart | | allopathy | Drug | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|791.7 MB| |Dependencies:|embeddings_clinical_medium| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English image_classifier_vit_deit_base_patch16_224 ViTForImageClassification from facebook author: John Snow Labs name: image_classifier_vit_deit_base_patch16_224 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_deit_base_patch16_224` is a English model originally trained by facebook. ## Predicted Entities `turnstile`, `damselfly`, `mixing bowl`, `sea snake`, `cockroach, roach`, `buckle`, `beer glass`, `bulbul`, `lumbermill, sawmill`, `whippet`, `Australian terrier`, `television, television system`, `hoopskirt, crinoline`, `horse cart, horse-cart`, `guillotine`, `malamute, malemute, Alaskan malamute`, `coyote, prairie wolf, brush wolf, Canis latrans`, `colobus, colobus monkey`, `hognose snake, puff adder, sand viper`, `sock`, `burrito`, `printer`, `bathing cap, swimming cap`, `chiton, coat-of-mail shell, sea cradle, polyplacophore`, `Rottweiler`, `cello, violoncello`, `pitcher, ewer`, `computer keyboard, keypad`, `bow`, `peacock`, `ballplayer, baseball player`, `refrigerator, icebox`, `solar dish, solar collector, solar furnace`, `passenger car, coach, carriage`, `African chameleon, Chamaeleo chamaeleon`, `oboe, hautboy, hautbois`, `toyshop`, `Leonberg`, `howler monkey, howler`, `bluetick`, `African elephant, Loxodonta africana`, `American lobster, Northern lobster, Maine lobster, Homarus americanus`, `combination lock`, `black-and-tan coonhound`, `bonnet, poke bonnet`, `harvester, reaper`, `Appenzeller`, `iron, smoothing iron`, `electric locomotive`, `lycaenid, lycaenid butterfly`, `sandbar, sand bar`, `Cardigan, Cardigan Welsh corgi`, `pencil sharpener`, `jean, blue jean, denim`, `backpack, back pack, knapsack, packsack, rucksack, haversack`, `monitor`, `ice cream, icecream`, `apiary, bee house`, `water jug`, `American coot, marsh hen, mud hen, water hen, Fulica americana`, `ground beetle, carabid beetle`, `jigsaw puzzle`, `ant, emmet, pismire`, `wreck`, `kuvasz`, `gyromitra`, `Ibizan hound, Ibizan Podenco`, `brown bear, bruin, Ursus arctos`, `bolo tie, bolo, bola tie, bola`, `Pembroke, Pembroke Welsh corgi`, `French bulldog`, `prison, prison house`, `ballpoint, ballpoint pen, ballpen, Biro`, `stage`, `airliner`, `dogsled, dog sled, dog sleigh`, `redshank, Tringa totanus`, `menu`, `Indian cobra, Naja naja`, `swab, swob, mop`, `window screen`, `brain coral`, `artichoke, globe artichoke`, `loupe, jeweler's loupe`, `loudspeaker, speaker, speaker unit, loudspeaker system, speaker system`, `panpipe, pandean pipe, syrinx`, `wok`, `croquet ball`, `plate`, `scoreboard`, `Samoyed, Samoyede`, `ocarina, sweet potato`, `beaver`, `borzoi, Russian wolfhound`, `horizontal bar, high bar`, `stretcher`, `seat belt, seatbelt`, `obelisk`, `forklift`, `feather boa, boa`, `frying pan, frypan, skillet`, `barbershop`, `hamper`, `face powder`, `Siamese cat, Siamese`, `ladle`, `dingo, warrigal, warragal, Canis dingo`, `mountain tent`, `head cabbage`, `echidna, spiny anteater, anteater`, `Polaroid camera, Polaroid Land camera`, `dumbbell`, `espresso`, `notebook, notebook computer`, `Norfolk terrier`, `binoculars, field glasses, opera glasses`, `carpenter's kit, tool kit`, `moving van`, `catamaran`, `tiger beetle`, `bikini, two-piece`, `Siberian husky`, `studio couch, day bed`, `bulletproof vest`, `lawn mower, mower`, `promontory, headland, head, foreland`, `soap dispenser`, `vulture`, `dam, dike, dyke`, `brambling, Fringilla montifringilla`, `toilet tissue, toilet paper, bathroom tissue`, `ringlet, ringlet butterfly`, `tiger cat`, `mobile home, manufactured home`, `Norwich terrier`, `little blue heron, Egretta caerulea`, `English setter`, `Tibetan mastiff`, `rocking chair, rocker`, `mask`, `maze, labyrinth`, `bookcase`, `viaduct`, `sweatshirt`, `plow, plough`, `basenji`, `typewriter keyboard`, `Windsor tie`, `coral fungus`, `desktop computer`, `Kerry blue terrier`, `Angora, Angora rabbit`, `can opener, tin opener`, `shield, buckler`, `triumphal arch`, `horned viper, cerastes, sand viper, horned asp, Cerastes cornutus`, `miniature schnauzer`, `tape player`, `jaguar, panther, Panthera onca, Felis onca`, `hook, claw`, `file, file cabinet, filing cabinet`, `chime, bell, gong`, `shower curtain`, `window shade`, `acoustic guitar`, `gas pump, gasoline pump, petrol pump, island dispenser`, `cicada, cicala`, `Petri dish`, `paintbrush`, `banana`, `chickadee`, `mountain bike, all-terrain bike, off-roader`, `lighter, light, igniter, ignitor`, `oil filter`, `cab, hack, taxi, taxicab`, `Christmas stocking`, `rugby ball`, `black widow, Latrodectus mactans`, `bustard`, `fiddler crab`, `web site, website, internet site, site`, `chocolate sauce, chocolate syrup`, `chainlink fence`, `fireboat`, `cocktail shaker`, `airship, dirigible`, `projectile, missile`, `bagel, beigel`, `screwdriver`, `oystercatcher, oyster catcher`, `pot, flowerpot`, `water bottle`, `Loafer`, `drumstick`, `soccer ball`, `cairn, cairn terrier`, `padlock`, `tow truck, tow car, wrecker`, `bloodhound, sleuthhound`, `punching bag, punch bag, punching ball, punchball`, `great grey owl, great gray owl, Strix nebulosa`, `scale, weighing machine`, `trench coat`, `briard`, `cheetah, chetah, Acinonyx jubatus`, `entertainment center`, `Boston bull, Boston terrier`, `Arabian camel, dromedary, Camelus dromedarius`, `steam locomotive`, `coil, spiral, volute, whorl, helix`, `plane, carpenter's plane, woodworking plane`, `gondola`, `spider web, spider's web`, `bathtub, bathing tub, bath, tub`, `pelican`, `miniature poodle`, `cowboy boot`, `perfume, essence`, `lakeside, lakeshore`, `timber wolf, grey wolf, gray wolf, Canis lupus`, `moped`, `sunscreen, sunblock, sun blocker`, `Brabancon griffon`, `puffer, pufferfish, blowfish, globefish`, `lifeboat`, `pool table, billiard table, snooker table`, `Bouvier des Flandres, Bouviers des Flandres`, `Pomeranian`, `theater curtain, theatre curtain`, `marimba, xylophone`, `baboon`, `vacuum, vacuum cleaner`, `pill bottle`, `pick, plectrum, plectron`, `hen`, `American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier`, `digital watch`, `pier`, `oxygen mask`, `Tibetan terrier, chrysanthemum dog`, `ostrich, Struthio camelus`, `water ouzel, dipper`, `drilling platform, offshore rig`, `magnetic compass`, `throne`, `butternut squash`, `minibus`, `EntleBucher`, `carousel, carrousel, merry-go-round, roundabout, whirligig`, `hot pot, hotpot`, `rain barrel`, `wood rabbit, cottontail, cottontail rabbit`, `miniature pinscher`, `partridge`, `three-toed sloth, ai, Bradypus tridactylus`, `English springer, English springer spaniel`, `corkscrew, bottle screw`, `fur coat`, `robin, American robin, Turdus migratorius`, `dowitcher`, `ruddy turnstone, Arenaria interpres`, `water snake`, `stove`, `Great Pyrenees`, `soft-coated wheaten terrier`, `carbonara`, `snail`, `breastplate, aegis, egis`, `wolf spider, hunting spider`, `hatchet`, `CD player`, `axolotl, mud puppy, Ambystoma mexicanum`, `pomegranate`, `poncho`, `leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea`, `lorikeet`, `spatula`, `jay`, `platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus`, `stethoscope`, `flagpole, flagstaff`, `coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch`, `agama`, `red wolf, maned wolf, Canis rufus, Canis niger`, `beaker`, `eft`, `pretzel`, `brassiere, bra, bandeau`, `frilled lizard, Chlamydosaurus kingi`, `joystick`, `goldfish, Carassius auratus`, `fig`, `maypole`, `caldron, cauldron`, `admiral`, `impala, Aepyceros melampus`, `spotted salamander, Ambystoma maculatum`, `syringe`, `hog, pig, grunter, squealer, Sus scrofa`, `handkerchief, hankie, hanky, hankey`, `tarantula`, `cheeseburger`, `pinwheel`, `sax, saxophone`, `dung beetle`, `broccoli`, `cassette player`, `milk can`, `traffic light, traffic signal, stoplight`, `shovel`, `sarong`, `tabby, tabby cat`, `parallel bars, bars`, `ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle`, `quill, quill pen`, `giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca`, `steel drum`, `quail`, `Blenheim spaniel`, `wig`, `hamster`, `ice lolly, lolly, lollipop, popsicle`, `seashore, coast, seacoast, sea-coast`, `chest`, `worm fence, snake fence, snake-rail fence, Virginia fence`, `missile`, `beer bottle`, `yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum`, `breakwater, groin, groyne, mole, bulwark, seawall, jetty`, `white wolf, Arctic wolf, Canis lupus tundrarum`, `guacamole`, `porcupine, hedgehog`, `trolleybus, trolley coach, trackless trolley`, `greenhouse, nursery, glasshouse`, `trimaran`, `Italian greyhound`, `potter's wheel`, `jacamar`, `wallet, billfold, notecase, pocketbook`, `Lakeland terrier`, `green lizard, Lacerta viridis`, `indigo bunting, indigo finch, indigo bird, Passerina cyanea`, `green mamba`, `walking stick, walkingstick, stick insect`, `crossword puzzle, crossword`, `eggnog`, `barrow, garden cart, lawn cart, wheelbarrow`, `remote control, remote`, `bicycle-built-for-two, tandem bicycle, tandem`, `wool, woolen, woollen`, `black grouse`, `abaya`, `marmoset`, `golf ball`, `jeep, landrover`, `Mexican hairless`, `dishwasher, dish washer, dishwashing machine`, `jersey, T-shirt, tee shirt`, `planetarium`, `goose`, `mailbox, letter box`, `capuchin, ringtail, Cebus capucinus`, `marmot`, `orangutan, orang, orangutang, Pongo pygmaeus`, `coffeepot`, `ambulance`, `shopping basket`, `pop bottle, soda bottle`, `red fox, Vulpes vulpes`, `crash helmet`, `street sign`, `affenpinscher, monkey pinscher, monkey dog`, `Arctic fox, white fox, Alopex lagopus`, `sidewinder, horned rattlesnake, Crotalus cerastes`, `ruffed grouse, partridge, Bonasa umbellus`, `muzzle`, `measuring cup`, `canoe`, `reflex camera`, `fox squirrel, eastern fox squirrel, Sciurus niger`, `French loaf`, `killer whale, killer, orca, grampus, sea wolf, Orcinus orca`, `dial telephone, dial phone`, `thimble`, `bubble`, `vizsla, Hungarian pointer`, `running shoe`, `mailbag, postbag`, `radio telescope, radio reflector`, `piggy bank, penny bank`, `Chihuahua`, `chambered nautilus, pearly nautilus, nautilus`, `Airedale, Airedale terrier`, `kimono`, `green snake, grass snake`, `rubber eraser, rubber, pencil eraser`, `upright, upright piano`, `orange`, `revolver, six-gun, six-shooter`, `ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin`, `drum, membranophone, tympan`, `Dungeness crab, Cancer magister`, `lipstick, lip rouge`, `gong, tam-tam`, `fountain`, `tub, vat`, `malinois`, `sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita`, `German short-haired pointer`, `apron`, `Irish setter, red setter`, `dishrag, dishcloth`, `school bus`, `candle, taper, wax light`, `bib`, `cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM`, `power drill`, `English foxhound`, `miniskirt, mini`, `swing`, `slug`, `hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa`, `rifle`, `Saluki, gazelle hound`, `Sealyham terrier, Sealyham`, `bullet train, bullet`, `hyena, hyaena`, `ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus`, `toy terrier`, `goblet`, `safe`, `cup`, `electric guitar`, `red wine`, `restaurant, eating house, eating place, eatery`, `wall clock`, `washbasin, handbasin, washbowl, lavabo, wash-hand basin`, `red-breasted merganser, Mergus serrator`, `crate`, `banded gecko`, `hippopotamus, hippo, river horse, Hippopotamus amphibius`, `tick`, `tripod`, `sombrero`, `desk`, `sea slug, nudibranch`, `racer, race car, racing car`, `pizza, pizza pie`, `dining table, board`, `Saint Bernard, St Bernard`, `komondor`, `electric ray, crampfish, numbfish, torpedo`, `prairie chicken, prairie grouse, prairie fowl`, `coffee mug`, `hammer`, `golfcart, golf cart`, `unicycle, monocycle`, `bison`, `soup bowl`, `rapeseed`, `golden retriever`, `plastic bag`, `grey fox, gray fox, Urocyon cinereoargenteus`, `water tower`, `house finch, linnet, Carpodacus mexicanus`, `barbell`, `hair slide`, `tiger, Panthera tigris`, `black-footed ferret, ferret, Mustela nigripes`, `meat loaf, meatloaf`, `hand blower, blow dryer, blow drier, hair dryer, hair drier`, `overskirt`, `gibbon, Hylobates lar`, `Gila monster, Heloderma suspectum`, `toucan`, `snowmobile`, `pencil box, pencil case`, `scuba diver`, `cloak`, `Sussex spaniel`, `otter`, `Greater Swiss Mountain dog`, `great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias`, `torch`, `magpie`, `tiger shark, Galeocerdo cuvieri`, `wing`, `Border collie`, `bell cote, bell cot`, `sea anemone, anemone`, `teapot`, `sea urchin`, `screen, CRT screen`, `bookshop, bookstore, bookstall`, `oscilloscope, scope, cathode-ray oscilloscope, CRO`, `crib, cot`, `police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria`, `hartebeest`, `manhole cover`, `iPod`, `rock python, rock snake, Python sebae`, `nipple`, `suspension bridge`, `safety pin`, `sea lion`, `cougar, puma, catamount, mountain lion, painter, panther, Felis concolor`, `mantis, mantid`, `wardrobe, closet, press`, `projector`, `Granny Smith`, `diamondback, diamondback rattlesnake, Crotalus adamanteus`, `pirate, pirate ship`, `espresso maker`, `African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus`, `cradle`, `common newt, Triturus vulgaris`, `tricycle, trike, velocipede`, `bobsled, bobsleigh, bob`, `thunder snake, worm snake, Carphophis amoenus`, `thresher, thrasher, threshing machine`, `banjo`, `armadillo`, `pajama, pyjama, pj's, jammies`, `ski`, `Maltese dog, Maltese terrier, Maltese`, `leafhopper`, `book jacket, dust cover, dust jacket, dust wrapper`, `silky terrier, Sydney silky`, `Shih-Tzu`, `wallaby, brush kangaroo`, `cardigan`, `sturgeon`, `freight car`, `home theater, home theatre`, `sundial`, `African crocodile, Nile crocodile, Crocodylus niloticus`, `odometer, hodometer, mileometer, milometer`, `sliding door`, `vine snake`, `West Highland white terrier`, `mongoose`, `hornbill`, `beagle`, `European gallinule, Porphyrio porphyrio`, `submarine, pigboat, sub, U-boat`, `Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis`, `cock`, `pedestal, plinth, footstall`, `accordion, piano accordion, squeeze box`, `gown`, `lynx, catamount`, `guenon, guenon monkey`, `Walker hound, Walker foxhound`, `standard schnauzer`, `reel`, `hip, rose hip, rosehip`, `grasshopper, hopper`, `Dutch oven`, `stone wall`, `hard disc, hard disk, fixed disk`, `snow leopard, ounce, Panthera uncia`, `shopping cart`, `digital clock`, `hourglass`, `Border terrier`, `Old English sheepdog, bobtail`, `academic gown, academic robe, judge's robe`, `spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish`, `spotlight, spot`, `dome`, `barn spider, Araneus cavaticus`, `bee eater`, `basketball`, `cliff dwelling`, `folding chair`, `isopod`, `Doberman, Doberman pinscher`, `bittern`, `sunglasses, dark glasses, shades`, `picket fence, paling`, `Crock Pot`, `ibex, Capra ibex`, `neck brace`, `cardoon`, `cassette`, `amphibian, amphibious vehicle`, `minivan`, `analog clock`, `trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi`, `yurt`, `cliff, drop, drop-off`, `Bernese mountain dog`, `teddy, teddy bear`, `sloth bear, Melursus ursinus, Ursus ursinus`, `bassoon`, `toaster`, `ptarmigan`, `Gordon setter`, `night snake, Hypsiglena torquata`, `grand piano, grand`, `purse`, `clumber, clumber spaniel`, `shoji`, `hair spray`, `maillot`, `knee pad`, `space heater`, `bottlecap`, `chiffonier, commode`, `chain saw, chainsaw`, `sulphur butterfly, sulfur butterfly`, `pay-phone, pay-station`, `kelpie`, `mouse, computer mouse`, `car wheel`, `cornet, horn, trumpet, trump`, `container ship, containership, container vessel`, `matchstick`, `scabbard`, `American black bear, black bear, Ursus americanus, Euarctos americanus`, `langur`, `rock crab, Cancer irroratus`, `lionfish`, `speedboat`, `black stork, Ciconia nigra`, `knot`, `disk brake, disc brake`, `mosquito net`, `white stork, Ciconia ciconia`, `abacus`, `titi, titi monkey`, `grocery store, grocery, food market, market`, `waffle iron`, `pickelhaube`, `wooden spoon`, `Norwegian elkhound, elkhound`, `earthstar`, `sewing machine`, `balance beam, beam`, `potpie`, `chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour`, `Staffordshire bullterrier, Staffordshire bull terrier`, `switch, electric switch, electrical switch`, `dhole, Cuon alpinus`, `paddle, boat paddle`, `limousine, limo`, `Shetland sheepdog, Shetland sheep dog, Shetland`, `space bar`, `library`, `paddlewheel, paddle wheel`, `alligator lizard`, `Band Aid`, `Persian cat`, `bull mastiff`, `tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui`, `sports car, sport car`, `football helmet`, `laptop, laptop computer`, `lens cap, lens cover`, `tennis ball`, `violin, fiddle`, `lab coat, laboratory coat`, `cinema, movie theater, movie theatre, movie house, picture palace`, `weasel`, `bow tie, bow-tie, bowtie`, `macaw`, `dough`, `whiskey jug`, `microphone, mike`, `spoonbill`, `bassinet`, `mud turtle`, `velvet`, `warthog`, `plunger, plumber's helper`, `dugong, Dugong dugon`, `honeycomb`, `badger`, `dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk`, `bee`, `doormat, welcome mat`, `fountain pen`, `giant schnauzer`, `assault rifle, assault gun`, `limpkin, Aramus pictus`, `siamang, Hylobates syndactylus, Symphalangus syndactylus`, `albatross, mollymawk`, `confectionery, confectionary, candy store`, `harp`, `parachute, chute`, `barrel, cask`, `tank, army tank, armored combat vehicle, armoured combat vehicle`, `collie`, `kite`, `puck, hockey puck`, `stupa, tope`, `buckeye, horse chestnut, conker`, `patio, terrace`, `broom`, `Dandie Dinmont, Dandie Dinmont terrier`, `scorpion`, `agaric`, `balloon`, `bucket, pail`, `squirrel monkey, Saimiri sciureus`, `Eskimo dog, husky`, `zebra`, `garter snake, grass snake`, `indri, indris, Indri indri, Indri brevicaudatus`, `tractor`, `guinea pig, Cavia cobaya`, `maraca`, `red-backed sandpiper, dunlin, Erolia alpina`, `bullfrog, Rana catesbeiana`, `trilobite`, `Japanese spaniel`, `gorilla, Gorilla gorilla`, `monastery`, `centipede`, `terrapin`, `llama`, `long-horned beetle, longicorn, longicorn beetle`, `boxer`, `curly-coated retriever`, `mortar`, `hammerhead, hammerhead shark`, `goldfinch, Carduelis carduelis`, `garden spider, Aranea diademata`, `stopwatch, stop watch`, `grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus`, `leaf beetle, chrysomelid`, `birdhouse`, `king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica`, `stole`, `bell pepper`, `radiator`, `flatworm, platyhelminth`, `mushroom`, `Scotch terrier, Scottish terrier, Scottie`, `liner, ocean liner`, `toilet seat`, `lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens`, `zucchini, courgette`, `harvestman, daddy longlegs, Phalangium opilio`, `Newfoundland, Newfoundland dog`, `flamingo`, `whiptail, whiptail lizard`, `geyser`, `cleaver, meat cleaver, chopper`, `sea cucumber, holothurian`, `American egret, great white heron, Egretta albus`, `parking meter`, `beacon, lighthouse, beacon light, pharos`, `coucal`, `motor scooter, scooter`, `mitten`, `cannon`, `weevil`, `megalith, megalithic structure`, `stinkhorn, carrion fungus`, `ear, spike, capitulum`, `box turtle, box tortoise`, `snowplow, snowplough`, `tench, Tinca tinca`, `modem`, `tobacco shop, tobacconist shop, tobacconist`, `barn`, `skunk, polecat, wood pussy`, `African grey, African gray, Psittacus erithacus`, `Madagascar cat, ring-tailed lemur, Lemur catta`, `holster`, `barometer`, `sleeping bag`, `washer, automatic washer, washing machine`, `recreational vehicle, RV, R.V.`, `drake`, `tray`, `butcher shop, meat market`, `china cabinet, china closet`, `medicine chest, medicine cabinet`, `photocopier`, `Yorkshire terrier`, `starfish, sea star`, `racket, racquet`, `park bench`, `Labrador retriever`, `whistle`, `clog, geta, patten, sabot`, `volcano`, `quilt, comforter, comfort, puff`, `leopard, Panthera pardus`, `cauliflower`, `swimming trunks, bathing trunks`, `American chameleon, anole, Anolis carolinensis`, `alp`, `mortarboard`, `barracouta, snoek`, `cocker spaniel, English cocker spaniel, cocker`, `space shuttle`, `beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon`, `harmonica, mouth organ, harp, mouth harp`, `gasmask, respirator, gas helmet`, `wombat`, `Model T`, `wild boar, boar, Sus scrofa`, `hermit crab`, `flat-coated retriever`, `rotisserie`, `jinrikisha, ricksha, rickshaw`, `trifle`, `bannister, banister, balustrade, balusters, handrail`, `go-kart`, `bakery, bakeshop, bakehouse`, `ski mask`, `dock, dockage, docking facility`, `Egyptian cat`, `oxcart`, `redbone`, `shoe shop, shoe-shop, shoe store`, `convertible`, `ox`, `crayfish, crawfish, crawdad, crawdaddy`, `cowboy hat, ten-gallon hat`, `conch`, `spaghetti squash`, `toy poodle`, `saltshaker, salt shaker`, `microwave, microwave oven`, `triceratops`, `necklace`, `castle`, `streetcar, tram, tramcar, trolley, trolley car`, `eel`, `diaper, nappy, napkin`, `standard poodle`, `prayer rug, prayer mat`, `radio, wireless`, `crane`, `envelope`, `rule, ruler`, `gar, garfish, garpike, billfish, Lepisosteus osseus`, `spider monkey, Ateles geoffroyi`, `Irish wolfhound`, `German shepherd, German shepherd dog, German police dog, alsatian`, `umbrella`, `sunglass`, `aircraft carrier, carrier, flattop, attack aircraft carrier`, `water buffalo, water ox, Asiatic buffalo, Bubalus bubalis`, `jellyfish`, `groom, bridegroom`, `tree frog, tree-frog`, `steel arch bridge`, `lemon`, `pickup, pickup truck`, `vault`, `groenendael`, `baseball`, `junco, snowbird`, `maillot, tank suit`, `gazelle`, `jack-o'-lantern`, `military uniform`, `slide rule, slipstick`, `wire-haired fox terrier`, `acorn squash`, `electric fan, blower`, `Brittany spaniel`, `chimpanzee, chimp, Pan troglodytes`, `pillow`, `binder, ring-binder`, `schipperke`, `Afghan hound, Afghan`, `plate rack`, `car mirror`, `hand-held computer, hand-held microcomputer`, `papillon`, `schooner`, `Bedlington terrier`, `cellular telephone, cellular phone, cellphone, cell, mobile phone`, `altar`, `Chesapeake Bay retriever`, `cabbage butterfly`, `polecat, fitch, foulmart, foumart, Mustela putorius`, `comic book`, `French horn, horn`, `daisy`, `organ, pipe organ`, `mashed potato`, `acorn`, `fly`, `chain`, `American alligator, Alligator mississipiensis`, `mink`, `garbage truck, dustcart`, `totem pole`, `wine bottle`, `strawberry`, `cricket`, `European fire salamander, Salamandra salamandra`, `coral reef`, `Welsh springer spaniel`, `bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis`, `snorkel`, `bald eagle, American eagle, Haliaeetus leucocephalus`, `meerkat, mierkat`, `grille, radiator grille`, `nematode, nematode worm, roundworm`, `anemone fish`, `corn`, `loggerhead, loggerhead turtle, Caretta caretta`, `palace`, `suit, suit of clothes`, `pineapple, ananas`, `macaque`, `ping-pong ball`, `ram, tup`, `church, church building`, `koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus`, `hare`, `bath towel`, `strainer`, `yawl`, `otterhound, otter hound`, `table lamp`, `king snake, kingsnake`, `lotion`, `lion, king of beasts, Panthera leo`, `thatch, thatched roof`, `basset, basset hound`, `black and gold garden spider, Argiope aurantia`, `barber chair`, `proboscis monkey, Nasalis larvatus`, `consomme`, `Irish terrier`, `Irish water spaniel`, `common iguana, iguana, Iguana iguana`, `Weimaraner`, `Great Dane`, `pug, pug-dog`, `rhinoceros beetle`, `vase`, `brass, memorial tablet, plaque`, `kit fox, Vulpes macrotis`, `king penguin, Aptenodytes patagonica`, `vending machine`, `dalmatian, coach dog, carriage dog`, `rock beauty, Holocanthus tricolor`, `pole`, `cuirass`, `bolete`, `jackfruit, jak, jack`, `monarch, monarch butterfly, milkweed butterfly, Danaus plexippus`, `chow, chow chow`, `nail`, `packet`, `half track`, `Lhasa, Lhasa apso`, `boathouse`, `hay`, `valley, vale`, `slot, one-armed bandit`, `volleyball`, `carton`, `shower cap`, `tile roof`, `lacewing, lacewing fly`, `patas, hussar monkey, Erythrocebus patas`, `boa constrictor, Constrictor constrictor`, `black swan, Cygnus atratus`, `lampshade, lamp shade`, `mousetrap`, `crutch`, `vestment`, `Pekinese, Pekingese, Peke`, `tusker`, `warplane, military plane`, `sandal`, `screw`, `custard apple`, `Scottish deerhound, deerhound`, `spindle`, `keeshond`, `hummingbird`, `letter opener, paper knife, paperknife`, `cucumber, cuke`, `bearskin, busby, shako`, `fire engine, fire truck`, `trombone`, `ringneck snake, ring-necked snake, ring snake`, `sorrel`, `fire screen, fireguard`, `paper towel`, `flute, transverse flute`, `hotdog, hot dog, red hot`, `Indian elephant, Elephas maximus`, `mosque`, `stingray`, `Rhodesian ridgeback`, `four-poster` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_base_patch16_224_en_4.1.0_3.0_1660165615831.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_base_patch16_224_en_4.1.0_3.0_1660165615831.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_deit_base_patch16_224", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_deit_base_patch16_224", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_deit_base_patch16_224| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|324.7 MB| --- layout: model title: Financial 10K Filings NER author: John Snow Labs name: finner_10k_summary date: 2022-08-17 tags: [en, finance, ner, annual, reports, 10k, filings, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole financial report. Instead: - Split by paragraphs; - Use the `finclf_form_10k_summary_item` Text Classifier to select only these paragraphs; This Financial NER Model is aimed to process the first summary page of 10K filings and extract the information about the Company submitting the filing, trading data, address / phones, CFN, IRS, etc. ## Predicted Entities `ADDRESS`, `CFN`, `FISCAL_YEAR`, `IRS`, `ORG`, `PHONE`, `STATE`, `STOCK_EXCHANGE`, `TICKER`, `TITLE_CLASS`, `TITLE_CLASS_VALUE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_SEC10K_FIRSTPAGE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_10k_summary_en_1.0.0_3.2_1660732829888.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_10k_summary_en_1.0.0_3.2_1660732829888.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") \ .setCustomBounds(["\n\n"]) tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_finbert_pretrain_yiyanghkust","en")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_10k_summary","en","finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES AND EXCHANGE ACT OF 1934 For the annual period ended January 31, 2021 or TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 For the transition period from________to_______ Commission File Number: 001-38856 PAGERDUTY, INC. (Exact name of registrant as specified in its charter) Delaware 27-2793871 (State or other jurisdiction of incorporation or organization) (I.R.S. Employer Identification Number) 600 Townsend St., Suite 200, San Francisco, CA 94103 (844) 800-3889 (Address, including zip code, and telephone number, including area code, of registrant’s principal executive offices) Securities registered pursuant to Section 12(b) of the Act: Title of each class Trading symbol(s) Name of each exchange on which registered Common Stock, $0.000005 par value, PD New York Stock Exchange"""]]).toDF("text") result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("ticker"), F.expr("cols['1']['entity']").alias("label")).show(50, truncate = False) ```

## Results ```bash +----------------------------------------------+-----------------+ |ticker |label | +----------------------------------------------+-----------------+ |January 31, 2021 |FISCAL_YEAR | |001-38856 |CFN | |PAGERDUTY, INC |ORG | |Delaware |STATE | |27-2793871 |IRS | |600 Townsend St., Suite 200, San Francisco, CA|ADDRESS | |(844) 800-3889 |PHONE | |Common Stock |TITLE_CLASS | |$0.000005 |TITLE_CLASS_VALUE| |PD |TICKER | |New York Stock Exchange |STOCK_EXCHANGE | +----------------------------------------------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_10k_summary| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.1 MB| ## References Manual annotations on 10-K Filings ## Benchmarking ```bash label precision recall f1-score support B-TITLE_CLASS 1.00 1.00 1.00 15 I-TITLE_CLASS 1.00 1.00 1.00 21 B-ORG 0.84 0.66 0.74 62 I-ORG 0.88 0.76 0.82 93 B-STOCK_EXCHANGE 0.86 0.86 0.86 14 I-STOCK_EXCHANGE 0.98 0.98 0.98 50 B-PHONE 0.95 0.87 0.91 23 I-PHONE 0.95 1.00 0.98 60 B-STATE 0.89 0.85 0.87 20 B-IRS 1.00 0.88 0.93 16 B-ADDRESS 0.94 0.83 0.88 18 I-ADDRESS 0.92 0.97 0.94 144 B-TICKER 0.86 0.92 0.89 13 B-FISCAL_YEAR 0.96 0.88 0.92 50 I-FISCAL_YEAR 0.93 0.92 0.92 125 B-TITLE_CLASS_VALUE 1.00 0.93 0.97 15 B-CFN 0.92 1.00 0.96 12 micro avg 0.93 0.89 0.91 751 macro avg 0.84 0.81 0.82 751 weighted avg 0.92 0.89 0.91 751 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from sshleifer) author: John Snow Labs name: distilbert_qa_tiny_base_cased_distilled_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tiny-distilbert-base-cased-distilled-squad` is a English model originally trained by `sshleifer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_tiny_base_cased_distilled_squad_en_4.0.0_3.0_1654728916941.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_tiny_base_cased_distilled_squad_en_4.0.0_3.0_1654728916941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tiny_base_cased_distilled_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tiny_base_cased_distilled_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_tiny_cased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_tiny_base_cased_distilled_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|641.4 KB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sshleifer/tiny-distilbert-base-cased-distilled-squad --- layout: model title: Stop Words Cleaner for Latin author: John Snow Labs name: stopwords_la date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: la edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, la] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_la_la_2.5.4_2.4_1594742439769.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_la_la_2.5.4_2.4_1594742439769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_la", "la") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_la", "la") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Alius est esse regem Aquilonis, et de Anglis medicus et nives Ioannes dux in progressus medicinae anesthesia et hygiene."""] stopword_df = nlu.load('la.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Alius', metadata={'sentence': '0'}), Row(annotatorType='token', begin=10, end=13, result='esse', metadata={'sentence': '0'}), Row(annotatorType='token', begin=15, end=19, result='regem', metadata={'sentence': '0'}), Row(annotatorType='token', begin=21, end=29, result='Aquilonis', metadata={'sentence': '0'}), Row(annotatorType='token', begin=30, end=30, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_la| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|la| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English BertForQuestionAnswering model (from vuiseng9) author: John Snow Labs name: bert_qa_bert_l_squadv1.1_sl256 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-l-squadv1.1-sl256` is a English model orginally trained by `vuiseng9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_l_squadv1.1_sl256_en_4.0.0_3.0_1654536057484.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_l_squadv1.1_sl256_en_4.0.0_3.0_1654536057484.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_l_squadv1.1_sl256","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_l_squadv1.1_sl256","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.sl256.by_vuiseng9").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_l_squadv1.1_sl256| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vuiseng9/bert-l-squadv1.1-sl256 --- layout: model title: Bangla RoBERTa Embeddings (from neuralspace-reverie) author: John Snow Labs name: roberta_embeddings_indic_transformers_bn_roberta date: 2022-04-14 tags: [roberta, embeddings, bn, open_source] task: Embeddings language: bn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indic-transformers-bn-roberta` is a Bangla model orginally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_bn_roberta_bn_3.4.2_3.0_1649947557406.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_bn_roberta_bn_3.4.2_3.0_1649947557406.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indic_transformers_bn_roberta| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|312.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-bn-roberta - https://oscar-corpus.com/ --- layout: model title: Pipeline to Detect Genes and Human Phenotypes author: John Snow Labs name: ner_human_phenotype_gene_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, gene, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_gene_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_human_phenotype_gene_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_pipeline_en_3.4.1_3.0_1647867336282.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_pipeline_en_3.4.1_3.0_1647867336282.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_human_phenotype_gene_biobert_pipeline", "en", "clinical/models") pipeline.annotate("Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).") ``` ```scala val pipeline = new PretrainedPipeline("ner_human_phenotype_gene_biobert_pipeline", "en", "clinical/models") pipeline.annotate("Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phenotype_gene_biobert.pipeline").predict("""Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).""") ```

## Results ```bash +----------------+--------+ |chunks |entities| +----------------+--------+ |type |GENE | |polyhydramnios |HP | |polyuria |HP | |nephrocalcinosis|HP | |hypokalemia |HP | +----------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_gene_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: English BertForQuestionAnswering model (from xraychen) author: John Snow Labs name: bert_qa_mqa_unsupsim date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mqa-unsupsim` is a English model orginally trained by `xraychen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mqa_unsupsim_en_4.0.0_3.0_1654188398566.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mqa_unsupsim_en_4.0.0_3.0_1654188398566.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mqa_unsupsim","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_mqa_unsupsim","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.unsupsim.by_xraychen").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mqa_unsupsim| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/xraychen/mqa-unsupsim --- layout: model title: Pipeline to Detect Normalized Genes and Human Phenotypes (biobert) author: John Snow Labs name: ner_human_phenotype_go_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_go_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_human_phenotype_go_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_biobert_pipeline_en_4.3.0_3.2_1679315805636.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_biobert_pipeline_en_4.3.0_3.2_1679315805636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_human_phenotype_go_biobert_pipeline", "en", "clinical/models") text = '''Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_human_phenotype_go_biobert_pipeline", "en", "clinical/models") val text = "Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.phenotype_go_biobert.pipeline").predict("""Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_cuenb_mnli","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_cuenb_mnli","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_cuenb_mnli| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|468.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/joey234/cuenb-mnli - https://paperswithcode.com/sota?task=Text+Classification&dataset=GLUE+MNLI --- layout: model title: Fast Neural Machine Translation Model from Austro-Asiatic languages to English author: John Snow Labs name: opus_mt_aav_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, aav, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `aav` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_aav_en_xx_2.7.0_2.4_1609169255439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_aav_en_xx_2.7.0_2.4_1609169255439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_aav_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes (sbiobertresolve_icd10cm_slim_billable_hcc) author: John Snow Labs name: sbiobertresolve_icd10cm_slim_billable_hcc date: 2021-05-25 tags: [icd10cm, slim, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to ICD10 CM codes using sentence biobert embeddings. In this model, synonyms having low cosine similarity to unnormalized terms are dropped. It also returns the official resolution text within the brackets inside the metadata. The model is augmented with synonyms, and previous augmentations are flexed according to cosine distances to unnormalized terms (ground truths). ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for aux_label parameter in the metadata to get HCC status. The HCC status can be divided to get further information: billable status, hcc status, and hcc score.For example, in the example shared below the billable status is 1, hcc status is 1, and hcc score is 11. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.0.3_2.4_1621942329774.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.0.3_2.4_1621942329774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:---------------|:--------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|-------------------------------------------:|:----------------------------|:---------------------------------------------------------| | 0 | bladder cancer | C671 |[bladder cancer, dome [Malignant neoplasm of dome of bladder], cancer of the urinary bladder [Malignant neoplasm of bladder, unspecified], adenocarcinoma, bladder neck [Malignant neoplasm of bladder neck], cancer in situ of urinary bladder [Carcinoma in situ of bladder], cancer of the urinary bladder, ureteric orifice [Malignant neoplasm of ureteric orifice], tumor of bladder neck [Neoplasm of unspecified behavior of bladder], cancer of the urethra [Malignant neoplasm of urethra]]| [C671, C679, C675, D090, C676, D494, C680] | ['1', '1', '11'] | [0.0685, 0.0709, 0.0963, 0.0978, 0.1068, 0.1080, 0.1211] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: Legal No Waivers Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_no_waivers_bert date: 2023-03-05 tags: [en, legal, classification, clauses, no_waivers, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `No_Waivers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `No_Waivers`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_waivers_bert_en_1.0.0_3.0_1678050601610.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_waivers_bert_en_1.0.0_3.0_1678050601610.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_no_waivers_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[No_Waivers]| |[Other]| |[Other]| |[No_Waivers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_waivers_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support No_Waivers 0.90 0.98 0.94 54 Other 0.99 0.92 0.95 73 accuracy - - 0.94 127 macro-avg 0.94 0.95 0.94 127 weighted-avg 0.95 0.94 0.95 127 ``` --- layout: model title: English DistilBertForQuestionAnswering Small Cased model (from ncduy) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_finetuned_small date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-finetuned-squad-small` is a English model originally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_finetuned_small_en_4.3.0_3.0_1672766627900.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_finetuned_small_en_4.3.0_3.0_1672766627900.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_finetuned_small","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_finetuned_small","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_finetuned_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/distilbert-base-cased-distilled-squad-finetuned-squad-small --- layout: model title: French CamemBert Embeddings (from mbateman) author: John Snow Labs name: camembert_embeddings_mbateman_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `mbateman`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_mbateman_generic_model_fr_3.4.4_3.0_1653989653583.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_mbateman_generic_model_fr_3.4.4_3.0_1653989653583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_mbateman_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/mbateman/dummy-model --- layout: model title: English BertForQuestionAnswering model (from bioformers) author: John Snow Labs name: bert_qa_bioformer_cased_v1.0_squad1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bioformer-cased-v1.0-squad1` is a English model orginally trained by `bioformers`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bioformer_cased_v1.0_squad1_en_4.0.0_3.0_1654185774933.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bioformer_cased_v1.0_squad1_en_4.0.0_3.0_1654185774933.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bioformer_cased_v1.0_squad1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bioformer_cased_v1.0_squad1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bioformer.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bioformer_cased_v1.0_squad1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|158.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bioformers/bioformer-cased-v1.0-squad1 - https://rajpurkar.github.io/SQuAD-explorer - https://arxiv.org/pdf/1910.01108.pdf --- layout: model title: Abkhazian asr_speech_sprint_test TFWav2Vec2ForCTC from Mofe author: John Snow Labs name: pipeline_asr_speech_sprint_test date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_speech_sprint_test` is a Abkhazian model originally trained by Mofe. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_speech_sprint_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_speech_sprint_test_ab_4.2.0_3.0_1664021417370.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_speech_sprint_test_ab_4.2.0_3.0_1664021417370.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_speech_sprint_test', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_speech_sprint_test", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_speech_sprint_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|452.6 KB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Counterparts Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_counterparts_bert date: 2023-03-05 tags: [en, legal, classification, clauses, counterparts, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Counterparts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Counterparts`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_counterparts_bert_en_1.0.0_3.0_1678050553108.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_counterparts_bert_en_1.0.0_3.0_1678050553108.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_counterparts_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Counterparts]| |[Other]| |[Other]| |[Counterparts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_counterparts_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Counterparts 1.00 0.99 1.0 302 Other 0.99 1.00 1.0 338 accuracy - - 1.0 640 macro-avg 1.00 1.00 1.0 640 weighted-avg 1.00 1.00 1.0 640 ``` --- layout: model title: Portuguese RoBERTa Embeddings (from rdenadai) author: John Snow Labs name: roberta_embeddings_BR_BERTo date: 2022-04-14 tags: [roberta, embeddings, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `BR_BERTo` is a Portuguese model orginally trained by `rdenadai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_BR_BERTo_pt_3.4.2_3.0_1649947632437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_BR_BERTo_pt_3.4.2_3.0_1649947632437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_BR_BERTo| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|637.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/rdenadai/BR_BERTo - https://github.com/rdenadai/BR-BERTo --- layout: model title: Chinese BertForMaskedLM Base Cased model author: John Snow Labs name: bert_embeddings_base_chinese date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-chinese` is a Chinese model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_chinese_zh_4.2.4_3.0_1670016364528.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_chinese_zh_4.2.4_3.0_1670016364528.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_chinese","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_chinese","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_chinese| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/bert-base-chinese - https://aclanthology.org/2021.acl-long.330.pdf - https://dl.acm.org/doi/pdf/10.1145/3442188.3445922 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_12_h_128 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-12_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_128_zh_4.2.4_3.0_1670021529933.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_128_zh_4.2.4_3.0_1670021529933.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_12_h_128| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|20.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-12_H-128 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Legal Sick leave Clause Binary Classifier author: John Snow Labs name: legclf_sick_leave_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `sick-leave` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `sick-leave` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sick_leave_clause_en_1.0.0_3.2_1660124014656.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sick_leave_clause_en_1.0.0_3.2_1660124014656.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_sick_leave_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[sick-leave]| |[other]| |[other]| |[sick-leave]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sick_leave_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.99 0.98 80 sick-leave 0.97 0.93 0.95 42 accuracy - - 0.97 122 macro-avg 0.97 0.96 0.96 122 weighted-avg 0.97 0.97 0.97 122 ``` --- layout: model title: Marathi Bert Embeddings (from monsoon-nlp) author: John Snow Labs name: bert_embeddings_muril_adapted_local date: 2022-04-11 tags: [bert, embeddings, mr, open_source] task: Embeddings language: mr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `muril-adapted-local` is a Marathi model orginally trained by `monsoon-nlp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_mr_3.4.2_3.0_1649675136793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_mr_3.4.2_3.0_1649675136793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_muril_adapted_local| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|mr| |Size:|888.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/monsoon-nlp/muril-adapted-local - https://tfhub.dev/google/MuRIL/1 --- layout: model title: Detect Assertion Status (assertion_dl) - supports confidence scores author: John Snow Labs name: assertion_dl date: 2021-01-26 task: Assertion Status language: en nav_key: models edition: Healthcare NLP 2.7.2 spark_version: 2.4 tags: [assertion, en, licensed, clinical] supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Assign assertion status to clinical entities extracted by NER based on their context in the text. ## Predicted Entities `absent`, `present`, `conditional`, `associated_with_someone_else`, `hypothetical`, `possible`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/2.Clinical_Assertion_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_dl_en_2.7.2_2.4_1611647201607.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_dl_en_2.7.2_2.4_1611647201607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------------------------+---------+-----------+ |chunk |ner_label|assertion | +-----------------------------------------------------------+---------+-----------+ |the G-protein-activated inwardly rectifying potassium (GIRK|TREATMENT|conditional| |the genomicorganization |TREATMENT|present | |a candidate gene forType II diabetes mellitus |PROBLEM |present | |byapproximately |TREATMENT|present | |single nucleotide polymorphisms |TREATMENT|present | |aVal366Ala substitution |TREATMENT|present | |insertion/deletion |PROBLEM |present | |'Ourexpression studies |TEST |present | |the transcript in various humantissues |PROBLEM |present | |fat andskeletal muscle |PROBLEM |possible | |furtherstudies |PROBLEM |present | |the KCNJ9 protein |TREATMENT|present | |evaluation |TEST |possible | |Type II diabetes |PROBLEM |present | +-----------------------------------------------------------+---------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_dl| |Compatibility:|Spark NLP 2.7.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| ## Data Source Trained on i2b2 assertion data ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|:-----------------------------|------:|-----:|-----:|---------:|---------:|---------:| | 0 | absent | 791 | 47 | 80 | 0.943914 | 0.908152 | 0.925688 | | 1 | present | 2499 | 169 | 120 | 0.936657 | 0.954181 | 0.945338 | | 2 | conditional | 23 | 19 | 21 | 0.547619 | 0.522727 | 0.534884 | | 3 | associated_with_someone_else | 38 | 2 | 11 | 0.95 | 0.77551 | 0.853933 | | 4 | hypothetical | 163 | 19 | 21 | 0.895604 | 0.88587 | 0.89071 | | 5 | possible | 119 | 61 | 64 | 0.661111 | 0.650273 | 0.655647 | | 6 | Macro-average | 3633 | 317 | 317 | 0.822484 | 0.782786 | 0.802144 | | 7 | Micro-average | 3633 | 317 | 317 | 0.919747 | 0.919747 | 0.919747 | ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from peggyhuang) author: John Snow Labs name: roberta_qa_canard date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-canard` is a English model originally trained by `peggyhuang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_canard_en_4.3.0_3.0_1674219967411.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_canard_en_4.3.0_3.0_1674219967411.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_canard","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_canard","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_canard| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|465.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/peggyhuang/roberta-canard --- layout: model title: Detect Problems, Tests and Treatments (ner_healthcare) author: John Snow Labs name: ner_healthcare_en date: 2020-03-26 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.4.4 spark_version: 2.4 tags: [ner, en, licensed, clinical] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Pretrained named entity recognition deep learning model for healthcare. Includes Problem, Test and Treatment entities. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. {:.h2_title} ## Predicted Entities ``PROBLEM``, ``TEST``, ``TREATMENT``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_en_2.4.4_2.4_1585188313964.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_en_2.4.4_2.4_1585188313964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPython.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_healthcare", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([["A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG ."]]).toDF("text")) results = model.transform(data) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_healthcare_100d", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_healthcare", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq(A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .).toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python anatomy_pipeline = PretrainedPipeline("bert_token_classifier_ner_anatomy_pipeline", "en", "clinical/models") anatomy_pipeline.annotate("This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.") ``` ```scala val anatomy_pipeline = new PretrainedPipeline("bert_token_classifier_ner_anatomy_pipeline", "en", "clinical/models") anatomy_pipeline.annotate("This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist. Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.anatomy_pipeline").predict("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now. General: Well-developed female, in no acute distress, afebrile. HEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.Neck: No lymphadenopathy. Chest: Clear. Abdomen: Positive bowel sounds and soft. Dermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilroberta_base_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_distilroberta_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.distilled_base.by_twmkn9").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilroberta_base_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/twmkn9/distilroberta-base-squad2 --- layout: model title: Legal Duties Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_duties_bert date: 2023-03-05 tags: [en, legal, classification, clauses, duties, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Duties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Duties`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_duties_bert_en_1.0.0_3.0_1678050512891.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_duties_bert_en_1.0.0_3.0_1678050512891.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_duties_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Duties]| |[Other]| |[Other]| |[Duties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_duties_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Duties 0.92 0.92 0.92 39 Other 0.95 0.95 0.95 61 accuracy - - 0.94 100 macro-avg 0.94 0.94 0.94 100 weighted-avg 0.94 0.94 0.94 100 ``` --- layout: model title: Relation extraction between body parts and direction entities (ReDL). author: John Snow Labs name: redl_bodypart_direction_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like `Internal_organ_or_component`, `External_body_part_or_region` etc. and Direction entities like `upper`, `lower` in clinical texts. `1` : Shows the body part and direction entity are related, `0` : Shows the body part and direction entity are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_direction_biobert_en_2.7.3_2.4_1612447753332.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_direction_biobert_en_2.7.3_2.4_1612447753332.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|-----------------------------|---------------|-------------|------------|-----------------------------|-------------|-------------|---------------|------------| | 0 | 1 | Direction | 35 | 39 | upper | Internal_organ_or_component | 41 | 50 | brain stem | 0.9999989 | | 1 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 59 | 68 | cerebellum | 0.99992585 | | 2 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.9999999 | | 3 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 54 | 57 | left | 0.999811 | | 4 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 75 | 79 | right | 0.9998203 | | 5 | 1 | Direction | 54 | 57 | left | Internal_organ_or_component | 59 | 68 | cerebellum | 1.0 | | 6 | 0 | Direction | 54 | 57 | left | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.97616416 | | 7 | 0 | Internal_organ_or_component | 59 | 68 | cerebellum | Direction | 75 | 79 | right | 0.953046 | | 8 | 1 | Direction | 75 | 79 | right | Internal_organ_or_component | 81 | 93 | basil ganglia | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_bodypart_direction_biobert| |Compatibility:|Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Data Source Trained on an internal dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.856 0.873 0.865 153 1 0.986 0.984 0.985 1347 Avg. 0.921 0.929 0.925 ``` --- layout: model title: Pipeline to Extract Entities in Spanish Clinical Trial Abstracts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_clinical_trials_abstracts_pipeline date: 2023-03-20 tags: [es, clinical, licensed, token_classification, bert, ner] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_clinical_trials_abstracts](https://nlp.johnsnowlabs.com/2022/08/11/bert_token_classifier_ner_clinical_trials_abstracts_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_pipeline_es_4.3.0_3.2_1679298645358.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_pipeline_es_4.3.0_3.2_1679298645358.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_clinical_trials_abstracts_pipeline", "es", "clinical/models") text = '''Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_clinical_trials_abstracts_pipeline", "es", "clinical/models") val text = "Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------|--------:|------:|:------------|-------------:| | 0 | suplementación | 13 | 26 | PROC | 0.999993 | | 1 | ácido fólico | 32 | 43 | CHEM | 0.999753 | | 2 | niveles de homocisteína | 55 | 77 | PROC | 0.997803 | | 3 | hemodiálisis | 101 | 112 | PROC | 0.999993 | | 4 | hiperhomocisteinemia | 118 | 137 | DISO | 0.999995 | | 5 | niveles de homocisteína | 248 | 270 | PROC | 0.999988 | | 6 | tHcy | 279 | 282 | PROC | 0.999989 | | 7 | ácido fólico | 309 | 320 | CHEM | 0.999987 | | 8 | vitamina B6 | 324 | 334 | CHEM | 0.999967 | | 9 | pp | 337 | 338 | CHEM | 0.999889 | | 10 | diálisis | 388 | 395 | PROC | 0.999993 | | 11 | función residual | 398 | 414 | PROC | 0.999948 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_clinical_trials_abstracts_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|410.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Mapping Drugs With Their Corresponding Actions And Treatments author: John Snow Labs name: drug_action_treatment_mapper date: 2022-04-04 tags: [en, chunkmapping, chunkmapper, drug, action, treatment, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.4.2 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps drugs with their corresponding `action` and `treatment`. `action` refers to the function of the drug in various body systems, `treatment` refers to which disease the drug is used to treat ## Predicted Entities `action`, `treatment` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/drug_action_treatment_mapper_en_3.4.2_3.0_1649098201229.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/drug_action_treatment_mapper_en_3.4.2_3.0_1649098201229.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+------------------+--------------------------------------------------------------+ |Drug |Treats |Pharmaceutical Action | +---------+------------------+--------------------------------------------------------------+ |Aklis |Hyperlipidemia |Hypertension:::Diabetic Kidney Disease:::Cerebrovascular... | |Dermovate|Lupus |Discoid Lupus Erythematosus:::Empeines:::Psoriasis:::Eczema...| |Diprivan |Infection |Laryngitis:::Pneumonia:::Pharyngitis | |Proventil|Addison's Disease |Allergic Conjunctivitis:::Anemia:::Ankylosing Spondylitis | +---------+------------------+--------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|drug_action_treatment_mapper| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|8.7 MB| --- layout: model title: Translate English to Afrikaans Pipeline author: John Snow Labs name: translate_en_af date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, af, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `af` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_af_xx_2.7.0_2.4_1609689571052.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_af_xx_2.7.0_2.4_1609689571052.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_af", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_af", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.af').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_af| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from batterydata) author: John Snow Labs name: bert_qa_batteryonlybert_uncased_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `batteryonlybert-uncased-squad-v1` is a English model orginally trained by `batterydata`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_batteryonlybert_uncased_squad_v1_en_4.0.0_3.0_1654179355858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_batteryonlybert_uncased_squad_v1_en_4.0.0_3.0_1654179355858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_batteryonlybert_uncased_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_batteryonlybert_uncased_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_battery.bert.uncased_only_bert.by_batterydata").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_batteryonlybert_uncased_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/batterydata/batteryonlybert-uncased-squad-v1 - https://github.com/ShuHuang/batterybert --- layout: model title: Fast Neural Machine Translation Model from Xhosa to English author: John Snow Labs name: opus_mt_xh_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, xh, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `xh` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_xh_en_xx_2.7.0_2.4_1609164176872.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_xh_en_xx_2.7.0_2.4_1609164176872.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_xh_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from holtin) Squad author: John Snow Labs name: distilbert_qa_holtin_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `holtin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_holtin_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725474082.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_holtin_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725474082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_holtin_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_holtin_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased_v2.by_holtin").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_holtin_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/holtin/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Consents Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_consents_bert date: 2023-03-05 tags: [en, legal, classification, clauses, consents, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Consents` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Consents`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_consents_bert_en_1.0.0_3.0_1678050585575.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_consents_bert_en_1.0.0_3.0_1678050585575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_consents_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Consents]| |[Other]| |[Other]| |[Consents]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_consents_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Consents 0.81 0.94 0.87 49 Other 0.95 0.85 0.90 72 accuracy - - 0.88 121 macro-avg 0.88 0.89 0.88 121 weighted-avg 0.89 0.88 0.89 121 ``` --- layout: model title: English Named Entity Recognition (from dslim) author: John Snow Labs name: bert_ner_bert_base_NER date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-NER` is a English model orginally trained by `dslim`. ## Predicted Entities `LOC`, `PER`, `ORG`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_NER_en_3.4.2_3.0_1652096558277.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_NER_en_3.4.2_3.0_1652096558277.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_NER| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/dslim/bert-base-NER - https://www.aclweb.org/anthology/W03-0419.pdf - https://www.aclweb.org/anthology/W03-0419.pdf - https://arxiv.org/pdf/1810.04805 - https://github.com/google-research/bert/issues/223 --- layout: model title: English image_classifier_vit_electric_2 ViTForImageClassification from smc author: John Snow Labs name: image_classifier_vit_electric_2 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_electric_2` is a English model originally trained by smc. ## Predicted Entities `poles`, `transformers` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_electric_2_en_4.1.0_3.0_1660171547079.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_electric_2_en_4.1.0_3.0_1660171547079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_electric_2", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_electric_2", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_electric_2| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Urdu BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_ur_cased date: 2022-12-02 tags: [ur, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ur edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-ur-cased` is a Urdu model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_ur_cased_ur_4.2.4_3.0_1670019274115.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_ur_cased_ur_4.2.4_3.0_1670019274115.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_ur_cased","ur") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_ur_cased","ur") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_ur_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ur| |Size:|348.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-ur-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_nl12 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-nl12` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl12_en_4.3.0_3.0_1675117150157.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl12_en_4.3.0_3.0_1675117150157.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_nl12","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_nl12","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_nl12| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|802.0 MB| ## References - https://huggingface.co/google/t5-efficient-large-nl12 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Fast Neural Machine Translation Model from English to Hungarian author: John Snow Labs name: opus_mt_en_hu date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, hu, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `hu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_hu_xx_2.7.0_2.4_1609170999360.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_hu_xx_2.7.0_2.4_1609170999360.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_hu| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Pohnpeian Pipeline author: John Snow Labs name: translate_en_pon date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, pon, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `pon` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_pon_xx_2.7.0_2.4_1609690686455.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_pon_xx_2.7.0_2.4_1609690686455.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_pon", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_pon", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.pon').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_pon| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Rundi to English author: John Snow Labs name: opus_mt_run_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, run, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `run` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_run_en_xx_2.7.0_2.4_1609166179021.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_run_en_xx_2.7.0_2.4_1609166179021.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_run_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Modified_SciBERT_512 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Modified-SciBERT-512` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_SciBERT_512_en_4.0.0_3.0_1657108506933.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_SciBERT_512_en_4.0.0_3.0_1657108506933.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Modified_SciBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Modified-SciBERT-512 --- layout: model title: Financial Sentiment Analysis (Lithuanian) author: John Snow Labs name: finclf_bert_sentiment_analysis date: 2022-10-22 tags: [lt, legal, classification, sentiment, analysis, licensed] task: Text Classification language: lt edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Lithuanian Sentiment Analysis Text Classifier, which will retrieve if a text is either expression a Positive Emotion or a Negative one. ## Predicted Entities `POS`,`NEG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_bert_sentiment_analysis_lt_1.0.0_3.0_1666475378253.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_bert_sentiment_analysis_lt_1.0.0_3.0_1666475378253.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python # Test classifier in Spark NLP pipeline document_assembler = nlp.DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = nlp.Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') # Load newly trained classifier sequenceClassifier_loaded = finance.BertForSequenceClassification.pretrained("finclf_bert_sentiment_analysis", "lt", "finance/models")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = nlp.Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier_loaded ]) # Generating example example = spark.createDataFrame([["Pagalbos paraðiuto laukiantis verslas priemones vertina teigiamai tik yra keli „jeigu“"]]).toDF("text") result = pipeline.fit(example).transform(example) # Checking results result.select("text", "class.result").show(truncate=False) ```

## Results ```bash +---------------------------------------------------------------------------------------+------+ |text |result| +---------------------------------------------------------------------------------------+------+ |Pagalbos paraðiuto laukiantis verslas priemones vertina teigiamai tik yra keli „jeigu“|[POS] | +---------------------------------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_bert_sentiment_analysis| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|lt| |Size:|406.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References An in-house augmented version of [this dataset](https://www.kaggle.com/datasets/rokastrimaitis/lithuanian-financial-news-dataset-and-bigrams?select=dataset%28original%29.csv) removing NEU tag ## Benchmarking ```bash label precision recall f1-score support NEG 0.80 0.76 0.78 509 POS 0.90 0.92 0.91 1167 accuracy - - 0.87 1676 macro-avg 0.85 0.84 0.84 1676 weighted-avg 0.87 0.87 0.87 1676 ``` --- layout: model title: Pipeline to Classify Texts into TREC-6 Classes author: John Snow Labs name: bert_sequence_classifier_trec_coarse_pipeline date: 2022-02-23 tags: [bert_sequence, trec, coarse, bert, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_sequence_classifier_trec_coarse_en](https://nlp.johnsnowlabs.com/2021/11/06/bert_sequence_classifier_trec_coarse_en.html). The TREC dataset for question classification consists of open-domain, fact-based questions divided into broad semantic categories. You can check the official documentation of the dataset, entities, etc. [here](https://search.r-project.org/CRAN/refmans/textdata/html/dataset_trec.html). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_trec_coarse_pipeline_en_3.4.0_3.0_1645609565500.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_trec_coarse_pipeline_en_3.4.0_3.0_1645609565500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python trec_pipeline = PretrainedPipeline("bert_sequence_classifier_trec_coarse_pipeline", lang = "en") trec_pipeline.annotate("Germany is the largest country in Europe economically.") ``` ```scala val trec_pipeline = new PretrainedPipeline("bert_sequence_classifier_trec_coarse_pipeline", lang = "en") trec_pipeline.annotate("Germany is the largest country in Europe economically.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_only_classfn_twostage_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_only_classfn_twostage_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_only_classfn_twostage_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|463.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_only_classfn_twostage_epochs_1_shard_1_squad2.0 --- layout: model title: Pipeline to Detect PHI for Deidentification (Generic) author: John Snow Labs name: ner_deid_generic_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deid, de] task: Named Entity Recognition language: de edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_generic](https://nlp.johnsnowlabs.com/2022/01/06/ner_deid_generic_de.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_pipeline_de_3.4.1_3.0_1647888023955.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_pipeline_de_3.4.1_3.0_1647888023955.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_generic_pipeline", "de", "clinical/models") pipeline.annotate("Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhaus in Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen.") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_generic_pipeline", "de", "clinical/models") pipeline.annotate("Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhaus in Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen.") ``` {:.nlu-block} ```python import nlu nlu.load("de.med_ner.deid_generic.pipeline").predict("""Michael Berger wird am Morgen des 12 Dezember 2018 ins St. Elisabeth-Krankenhaus in Bad Kissingen eingeliefert. Herr Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen.""") ```

## Results ```bash +-----------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------+---------+ |Michael Berger |NAME | |12 Dezember 2018 |DATE | |St. Elisabeth-Krankenhausin Bad Kissingen|LOCATION | |Berger |NAME | |76 |AGE | +-----------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|de| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Fast Neural Machine Translation Model from Arabic to Spanish author: John Snow Labs name: opus_mt_ar_es date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, es, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: es {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_es_xx_3.1.0_2.4_1622550859700.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_es_xx_3.1.0_2.4_1622550859700.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from nlpunibo) author: John Snow Labs name: distilbert_qa_base_config2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert_base_config2` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config2_en_4.3.0_3.0_1672774448417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_config2_en_4.3.0_3.0_1672774448417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_config2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_config2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/distilbert_base_config2 --- layout: model title: Telugu Bert Embeddings author: John Snow Labs name: bert_embeddings_telugu_bertu date: 2022-04-11 tags: [bert, embeddings, te, open_source] task: Embeddings language: te edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `telugu_bertu` is a Telugu model orginally trained by `kuppuluri`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_telugu_bertu_te_3.4.2_3.0_1649675264476.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_telugu_bertu_te_3.4.2_3.0_1649675264476.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_telugu_bertu| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|te| |Size:|415.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/kuppuluri/telugu_bertu --- layout: model title: Translate English to Russian Pipeline author: John Snow Labs name: translate_en_ru date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ru, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ru` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ru_xx_2.7.0_2.4_1609687763987.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ru_xx_2.7.0_2.4_1609687763987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ru", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ru", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ru').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ru| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Lushai Pipeline author: John Snow Labs name: translate_en_lus date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, lus, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `lus` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_lus_xx_2.7.0_2.4_1609690402942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_lus_xx_2.7.0_2.4_1609690402942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_lus", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_lus", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.lus').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_lus| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Base Cased model (from mrm8488) author: John Snow Labs name: t5_base_finetuned_span_sentiment_extraction date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-finetuned-span-sentiment-extraction` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_span_sentiment_extraction_en_4.3.0_3.0_1675109003319.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_finetuned_span_sentiment_extraction_en_4.3.0_3.0_1675109003319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_finetuned_span_sentiment_extraction","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_finetuned_span_sentiment_extraction","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_finetuned_span_sentiment_extraction| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|905.9 MB| ## References - https://huggingface.co/mrm8488/t5-base-finetuned-span-sentiment-extraction - https://twitter.com/AND__SO - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://www.kaggle.com/c/tweet-sentiment-extraction - https://arxiv.org/pdf/1910.10683.pdf - https://www.kaggle.com/c/tweet-sentiment-extraction - https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb - https://github.com/enzoampil - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: English image_classifier_vit_Infrastructures ViTForImageClassification from drab author: John Snow Labs name: image_classifier_vit_Infrastructures date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Infrastructures` is a English model originally trained by drab. ## Predicted Entities `Cooling tower`, `Transmission grid`, `Wind turbines` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Infrastructures_en_4.1.0_3.0_1660167727547.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Infrastructures_en_4.1.0_3.0_1660167727547.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Infrastructures", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Infrastructures", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Infrastructures| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Kinyarwanda XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand date: 2022-08-01 tags: [rw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: rw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-kinyarwanda-finetuned-ner-kinyarwanda` is a Kinyarwanda model originally trained by `mbeukman`. ## Predicted Entities `PER`, `DATE`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659354149074.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659354149074.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand","rw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand","rw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_kinyarwanda_finetuned_ner_kinyarwand| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|rw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-kinyarwanda-finetuned-ner-kinyarwanda - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Detect Entities Related to Cancer Therapies author: John Snow Labs name: ner_oncology_therapy_wip date: 2022-09-30 tags: [licensed, clinical, oncology, en, ner, treatment] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to oncology therapies using granular labels, including mentions of treatments, posology information and line of therapy. Definitions of Predicted Entities: - `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. - `Chemotherapy`: Mentions of chemotherapy drugs, or unspecific words such as "chemotherapy". - `Cycle_Count`: The total number of cycles being administered of an oncological therapy (e.g. "5 cycles"). - `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. "day 5"). - `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. "third cycle"). - `Dosage`: The quantity prescribed by the physician for an active ingredient. - `Duration`: Words indicating the duration of a treatment (e.g. "for 2 weeks"). - `Frequency`: Words indicating the frequency of treatment administration (e.g. "daily" or "bid"). - `Hormonal_Therapy`: Mentions of hormonal drugs used to treat cancer, or unspecific words such as "hormonal therapy". - `Immunotherapy`: Mentions of immunotherapy drugs, or unspecific words such as "immunotherapy". - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Radiotherapy`: Terms that indicate the use of Radiotherapy. - `Radiation_Dose`: Dose used in radiotherapy. - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Route`: Words indicating the type of administration route (such as "PO" or "transdermal"). - `Targeted_Therapy`: Mentions of targeted therapy drugs, or unspecific words such as "targeted therapy". - `Unspecific_Therapy`: Terms that indicate a known cancer therapy but that is not specific to any other therapy entity (e.g. "chemoradiotherapy" or "adjuvant therapy"). ## Predicted Entities `Cancer_Surgery`, `Chemotherapy`, `Cycle_Count`, `Cycle_Day`, `Cycle_Number`, `Dosage`, `Duration`, `Frequency`, `Hormonal_Therapy`, `Immunotherapy`, `Line_Of_Therapy`, `Radiotherapy`, `Radiation_Dose`, `Response_To_Treatment`, `Route`, `Targeted_Therapy`, `Unspecific_Therapy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_wip_en_4.0.0_3.0_1664557936894.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_wip_en_4.0.0_3.0_1664557936894.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------------------------|:----------------------| | mastectomy | Cancer_Surgery | | axillary lymph node dissection | Cancer_Surgery | | radiotherapy | Radiotherapy | | recurred | Response_To_Treatment | | adriamycin | Chemotherapy | | 60 mg/m2 | Dosage | | cyclophosphamide | Chemotherapy | | 600 mg/m2 | Dosage | | six courses | Cycle_Count | | first line | Line_Of_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_therapy_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|869.8 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Cycle_Number 58.0 18.0 15.0 73.0 0.76 0.79 0.78 Response_To_Treatment 249.0 80.0 180.0 429.0 0.76 0.58 0.66 Cycle_Count 151.0 48.0 24.0 175.0 0.76 0.86 0.81 Unspecific_Therapy 167.0 88.0 67.0 234.0 0.65 0.71 0.68 Chemotherapy 535.0 30.0 83.0 618.0 0.95 0.87 0.90 Targeted_Therapy 144.0 9.0 35.0 179.0 0.94 0.80 0.87 Radiotherapy 188.0 17.0 34.0 222.0 0.92 0.85 0.88 Cancer_Surgery 526.0 60.0 119.0 645.0 0.90 0.82 0.85 Line_Of_Therapy 73.0 14.0 14.0 87.0 0.84 0.84 0.84 Hormonal_Therapy 95.0 1.0 21.0 116.0 0.99 0.82 0.90 Immunotherapy 90.0 58.0 21.0 111.0 0.61 0.81 0.69 Cycle_Day 149.0 33.0 34.0 183.0 0.82 0.81 0.82 Frequency 287.0 35.0 62.0 349.0 0.89 0.82 0.86 Route 82.0 17.0 15.0 97.0 0.83 0.85 0.84 Duration 399.0 95.0 148.0 547.0 0.81 0.73 0.77 Dosage 718.0 38.0 109.0 827.0 0.95 0.87 0.91 Radiation_Dose 84.0 15.0 12.0 96.0 0.85 0.88 0.86 macro_avg 3995.0 656.0 993.0 4988.0 0.84 0.81 0.82 micro_avg NaN NaN NaN NaN 0.86 0.80 0.83 ``` --- layout: model title: English image_classifier_vit_roomclassifier ViTForImageClassification from lazyturtl author: John Snow Labs name: image_classifier_vit_roomclassifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_roomclassifier` is a English model originally trained by lazyturtl. ## Predicted Entities `Kitchen`, `Bedroom`, `Bathroom`, `Livingroom`, `DinningRoom`, `Laundry room` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_roomclassifier_en_4.1.0_3.0_1660171006278.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_roomclassifier_en_4.1.0_3.0_1660171006278.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_roomclassifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_roomclassifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_roomclassifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: SDOH Environment Status Classification author: John Snow Labs name: bert_sequence_classifier_sdoh_environment_status date: 2022-12-18 tags: [en, clinical, sdoh, licensed, sequence_classification, environment_status, classifier] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model classifies related to environment situation such as any indication of housing, homeless or no related passage. A discharge summary was classified as True for the SDOH Environment if there was any indication of housing, False if the patient was homeless and None if there was no related passage. ## Predicted Entities `True`, `False`, `None` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_sdoh_environment_status_en_4.2.2_3.0_1671371837321.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_sdoh_environment_status_en_4.2.2_3.0_1671371837321.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_sdoh_environment_status", "en", "clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) sample_texts = ["The patient is a 29-year-old female with a history of renal transplant in 2097, who had worsening renal failure for the past several months. Her chief complaints were hypotension and seizure. months prior to admission and had been more hypertensive recently, requiring blood pressure medications. She was noted to have worsening renal function secondary to recent preeclampsia and her blood pressure control was thought to be secondary to renal failure.", "Mr Known lastname 19017 is a 66 year-old man with a PMHx of stage 4 COPD (FEV1 0.65L;FEV1/FVC 37% predicted in 4-14) on 4L home o2 with numerous hospitalizations for COPD exacerbations and intubation, hypertension, coronary artery disease, GERD who presents with SOB and CP. He is admitted to the ICU for management of dyspnea and hypotension.", "He was deemed Child's B in 2156-5-17 with ongoing ethanol abuse, admitted to Intensive Care Unit due to acute decompensation of chronic liver disease due to alcoholic hepatitis and Escherichia coli sepsis. after being hit in the head with the a bottle and dropping to the floor in the apartment. They had Trauma work him up including a head computerized tomography scan which was negative. He had abdominal pain for approximately one month with increasing abdominal girth, was noted to be febrile to 100 degrees on presentation and was tachycardiac 130, stable blood pressures. He was noted to have distended abdomen with diffuse tenderness computerized tomography scan of the abdomen which showed ascites and large nodule of the liver, splenomegaly, paraesophageal varices and loops of thickened bowel."] data = spark.createDataFrame(sample_texts, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_sdoh_environment_status", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val data = Seq("The patient is a 29-year-old female with a history of renal transplant in 2097, who had worsening renal failure for the past several months. Her chief complaints were hypotension and seizure. months prior to admission and had been more hypertensive recently, requiring blood pressure medications. She was noted to have worsening renal function secondary to recent preeclampsia and her blood pressure control was thought to be secondary to renal failure.") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_sequence.sdoh.environment_status").predict("""He was deemed Child's B in 2156-5-17 with ongoing ethanol abuse, admitted to Intensive Care Unit due to acute decompensation of chronic liver disease due to alcoholic hepatitis and Escherichia coli sepsis. after being hit in the head with the a bottle and dropping to the floor in the apartment. They had Trauma work him up including a head computerized tomography scan which was negative. He had abdominal pain for approximately one month with increasing abdominal girth, was noted to be febrile to 100 degrees on presentation and was tachycardiac 130, stable blood pressures. He was noted to have distended abdomen with diffuse tenderness computerized tomography scan of the abdomen which showed ascites and large nodule of the liver, splenomegaly, paraesophageal varices and loops of thickened bowel.""") ```

## Results ```bash +----------------------------------------------------------------------------------------------------+-------+ | text| result| +----------------------------------------------------------------------------------------------------+-------+ |The patient is a 29-year-old female with a history of renal transplant in 2097, who had worsening...| [None]| |Mr Known lastname 19017 is a 66 year-old man with a PMHx of stage 4 COPD (FEV1 0.65L;FEV1/FVC 37%...|[False]| |He was deemed Child's B in 2156-5-17 with ongoing ethanol abuse, admitted to Intensive Care Unit ...| [True]| +----------------------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_sdoh_environment_status| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|410.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## Benchmarking ```bash label precision recall f1-score support None 0.89 0.78 0.83 277 False 0.86 0.93 0.90 419 True 0.67 1.00 0.80 6 accuracy - - 0.87 702 macro-avg 0.81 0.90 0.84 702 weighted-avg 0.87 0.87 0.87 702 ``` --- layout: model title: Danish asr_wav2vec2_xls_r_300m_ftspeech TFWav2Vec2ForCTC from saattrupdan author: John Snow Labs name: asr_wav2vec2_xls_r_300m_ftspeech date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_ftspeech` is a Danish model originally trained by saattrupdan. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_ftspeech_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_ftspeech_da_4.2.0_3.0_1664101599815.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_ftspeech_da_4.2.0_3.0_1664101599815.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_ftspeech", "da")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_ftspeech", "da") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_ftspeech| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|da| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from tli8hf) author: John Snow Labs name: roberta_qa_unqover_large_news date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unqover-roberta-large-newsqa` is a English model originally trained by `tli8hf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_large_news_en_4.3.0_3.0_1674224676431.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_large_news_en_4.3.0_3.0_1674224676431.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_large_news","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_large_news","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_unqover_large_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/tli8hf/unqover-roberta-large-newsqa --- layout: model title: Financial Legal proceedings Item Binary Classifier author: John Snow Labs name: finclf_legal_proceedings_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `legal_proceedings` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `legal_proceedings` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_legal_proceedings_item_en_1.0.0_3.2_1660154442715.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_legal_proceedings_item_en_1.0.0_3.2_1660154442715.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_legal_proceedings_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[legal_proceedings]| |[other]| |[other]| |[legal_proceedings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_legal_proceedings_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support legal_proceedings 0.96 0.88 0.92 25 other 0.92 0.97 0.95 36 accuracy - - 0.93 61 macro-avg 0.94 0.93 0.93 61 weighted-avg 0.94 0.93 0.93 61 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from lordtt13) author: John Snow Labs name: t5_inshorts date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-inshorts` is a English model originally trained by `lordtt13`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_inshorts_en_4.3.0_3.0_1675124897561.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_inshorts_en_4.3.0_3.0_1675124897561.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_inshorts","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_inshorts","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_inshorts| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|927.0 MB| ## References - https://huggingface.co/lordtt13/t5-inshorts - https://arxiv.org/abs/1910.10683 - https://www.kaggle.com/shashichander009/inshorts-news-data - https://github.com/lordtt13/transformers-experiments/blob/master/Custom%20Tasks/fine-tune-t5-for-summarization.ipynb - https://github.com/lordtt13 - https://www.linkedin.com/in/tanmay-thakur-6bb5a9154/ --- layout: model title: English asr_wav2vec2_base_timit_demo_colab240 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab240 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab240` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab240_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab240_en_4.2.0_3.0_1664023948102.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab240_en_4.2.0_3.0_1664023948102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab240', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab240", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab240| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Russian T5ForConditionalGeneration Small Cased model (from cointegrated) author: John Snow Labs name: t5_rut5_small_normalizer date: 2023-01-30 tags: [ru, open_source, t5, tensorflow] task: Text Generation language: ru edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rut5-small-normalizer` is a Russian model originally trained by `cointegrated`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_rut5_small_normalizer_ru_4.3.0_3.0_1675106835622.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_rut5_small_normalizer_ru_4.3.0_3.0_1675106835622.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_rut5_small_normalizer","ru") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_rut5_small_normalizer","ru") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_rut5_small_normalizer| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ru| |Size:|277.8 MB| ## References - https://huggingface.co/cointegrated/rut5-small-normalizer - https://github.com/natasha/natasha - https://github.com/kmike/pymorphy2 - https://wortschatz.uni-leipzig.de/en/download/Russian --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from jgammack) author: John Snow Labs name: roberta_qa_jgammack_base_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad` is a English model originally trained by `jgammack`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_jgammack_base_squad_en_4.3.0_3.0_1674218670079.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_jgammack_base_squad_en_4.3.0_3.0_1674218670079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_jgammack_base_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_jgammack_base_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_jgammack_base_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jgammack/roberta-base-squad --- layout: model title: Pipeline to Detect Drug Information author: John Snow Labs name: ner_posology_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, drug, en] task: [Named Entity Recognition, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_posology_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_posology_biobert_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_POSOLOGY.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_biobert_pipeline_en_3.4.1_3.0_1647871826696.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_biobert_pipeline_en_3.4.1_3.0_1647871826696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_posology_biobert_pipeline", "en", "clinical/models") pipeline.fullAnnotate('The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.') ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_posology_biobert_pipeline", "en", "clinical/models") pipeline.fullAnnotate("The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology_biobert.pipeline").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunks |entities | +--------------+---------+ |insulin |DRUG | |Bactrim |DRUG | |for 14 days |DURATION | |Fragmin |DRUG | |5000 units |DOSAGE | |subcutaneously|ROUTE | |daily |FREQUENCY| |Xenaderm |DRUG | |topically |ROUTE | |b.i.d |FREQUENCY| |Lantus |DRUG | |40 units |DOSAGE | |subcutaneously|ROUTE | |at bedtime |FREQUENCY| |OxyContin |DRUG | |30 mg |STRENGTH | |p.o |ROUTE | |q.12 h |FREQUENCY| |folic acid |DRUG | |1 mg |STRENGTH | +--------------+---------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Pipeline to Detect Clinical Entities (ner_jsl) author: John Snow Labs name: ner_jsl_pipeline date: 2023-03-09 tags: [ner, licensed, en, clinical] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl](https://nlp.johnsnowlabs.com/2022/10/19/ner_jsl_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_pipeline_en_4.3.0_3.2_1678353833465.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_pipeline_en_4.3.0_3.2_1678353833465.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_jsl_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). Additionally, there is no side effect observed after Influenza vaccine. One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_jsl_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). Additionally, there is no side effect observed after Influenza vaccine. One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). Additionally, there is no side effect observed after Influenza vaccine. One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.997 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.9995 | | 2 | male | 38 | 41 | Gender | 0.9998 | | 3 | 2 days | 52 | 57 | Duration | 0.805 | | 4 | congestion | 62 | 71 | Symptom | 0.9049 | | 5 | mom | 75 | 77 | Gender | 0.9907 | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | 0.268133 | | 7 | nares | 135 | 139 | External_body_part_or_region | 0.7284 | | 8 | she | 147 | 149 | Gender | 0.9978 | | 9 | mild | 168 | 171 | Modifier | 0.7517 | | 10 | problems with his breathing while feeding | 173 | 213 | Symptom | 0.664583 | | 11 | perioral cyanosis | 237 | 253 | Symptom | 0.6869 | | 12 | retractions | 258 | 268 | Symptom | 0.9912 | | 13 | Influenza vaccine | 325 | 341 | Vaccine_Name | 0.833 | | 14 | One day ago | 344 | 354 | RelativeDate | 0.8667 | | 15 | mom | 357 | 359 | Gender | 0.9991 | | 16 | tactile temperature | 376 | 394 | Symptom | 0.3339 | | 17 | Tylenol | 417 | 423 | Drug_BrandName | 0.9988 | | 18 | Baby | 426 | 429 | Age | 0.9634 | | 19 | decreased p.o | 449 | 461 | Symptom | 0.75925 | | 20 | His | 472 | 474 | Gender | 0.9998 | | 21 | 20 minutes | 511 | 520 | Duration | 0.48575 | | 22 | 5 to 10 minutes | 531 | 545 | Duration | 0.526575 | | 23 | his | 560 | 562 | Gender | 0.988 | | 24 | respiratory congestion | 564 | 585 | Symptom | 0.6168 | | 25 | He | 588 | 589 | Gender | 0.9992 | | 26 | tired | 622 | 626 | Symptom | 0.8745 | | 27 | fussy | 641 | 645 | Symptom | 0.8509 | | 28 | over the past 2 days | 647 | 666 | RelativeDate | 0.60494 | | 29 | albuterol | 709 | 717 | Drug_Ingredient | 0.9876 | | 30 | ER | 743 | 744 | Clinical_Dept | 0.9974 | | 31 | His | 747 | 749 | Gender | 0.9996 | | 32 | urine output has also decreased | 751 | 781 | Symptom | 0.39878 | | 33 | he | 793 | 794 | Gender | 0.997 | | 34 | per 24 hours | 832 | 843 | Frequency | 0.462333 | | 35 | he | 850 | 851 | Gender | 0.9983 | | 36 | per 24 hours | 879 | 890 | Frequency | 0.562167 | | 37 | Mom | 893 | 895 | Gender | 0.9997 | | 38 | diarrhea | 908 | 915 | Symptom | 0.9956 | | 39 | His | 918 | 920 | Gender | 0.9997 | | 40 | bowel | 922 | 926 | Internal_organ_or_component | 0.9218 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Relation Extraction Between Body Parts and Direction Entities (ReDL) author: John Snow Labs name: redl_bodypart_direction_biobert date: 2021-06-01 tags: [licensed, en, clinical, relation_extraction] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like `Internal_organ_or_component`, `External_body_part_or_region` etc. and direction entities like `upper`, `lower` in clinical texts. `1` : Shows the body part and direction entity are related, `0` : Shows the body part and direction entity are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_direction_biobert_en_3.0.3_3.0_1622564511730.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_direction_biobert_en_3.0.3_3.0_1622564511730.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|-----------------------------|---------------|-------------|------------|-----------------------------|-------------|-------------|---------------|------------| | 0 | 1 | Direction | 35 | 39 | upper | Internal_organ_or_component | 41 | 50 | brain stem | 0.9999989 | | 1 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 59 | 68 | cerebellum | 0.99992585 | | 2 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.9999999 | | 3 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 54 | 57 | left | 0.999811 | | 4 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 75 | 79 | right | 0.9998203 | | 5 | 1 | Direction | 54 | 57 | left | Internal_organ_or_component | 59 | 68 | cerebellum | 1.0 | | 6 | 0 | Direction | 54 | 57 | left | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.97616416 | | 7 | 0 | Internal_organ_or_component | 59 | 68 | cerebellum | Direction | 75 | 79 | right | 0.953046 | | 8 | 1 | Direction | 75 | 79 | right | Internal_organ_or_component | 81 | 93 | basil ganglia | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_bodypart_direction_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on an internal dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.856 0.873 0.865 153 1 0.986 0.984 0.985 1347 Avg. 0.921 0.929 0.925 - ``` --- layout: model title: Clinical Deidentification author: John Snow Labs name: clinical_deidentification date: 2022-03-03 tags: [deidentification, en, licensed, pipeline, clinical] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_3.4.1_2.4_1646340071616.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_3.4.1_2.4_1646340071616.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification","en","clinical/models") val sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" val result =deid_pipeline .annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.clinical_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

## Results ```bash Masked with entity labels ------------------------------ Name : , Record date: , # . Dr. , ID, IP . He is a male was admitted to the for cystectomy on . Patient's VIN : , SSN , Driver's license . Phone , , , E-MAIL: . Masked with chars ------------------------------ Name : [**************], Record date: [********], # [****]. Dr. [********], ID[**********], IP [************]. He is a [*********] male was admitted to the [**********] for cystectomy on [******]. Patient's VIN : [***************], SSN [**********], Driver's license [*********]. Phone [************], [***************], [***********], E-MAIL: [*************]. Masked with fixed length chars ------------------------------ Name : ****, Record date: ****, # ****. Dr. ****, ID****, IP ****. He is a **** male was admitted to the **** for cystectomy on ****. Patient's VIN : ****, SSN ****, Driver's license ****. Phone ****, ****, ****, E-MAIL: ****. Obfuscated ------------------------------ Name : Berneta Phenes, Record date: 2093-03-14, # Y5003067. Dr. Dr Gaston Margo, IDOX:8976967, IP 001.001.001.001. He is a 91 male was admitted to the MADONNA REHABILITATION HOSPITAL for cystectomy on 07-22-1994. Patient's VIN : 5eeee44ffff555666, SSN 999-84-3686, Driver's license S99956482. Phone 74 617 042, 1407 west stassney lane, Edmonton, E-MAIL: Carliss@hotmail.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English asr_final_wav2vec2_urdu_asr_project TFWav2Vec2ForCTC from Raffay author: John Snow Labs name: pipeline_asr_final_wav2vec2_urdu_asr_project date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_final_wav2vec2_urdu_asr_project` is a English model originally trained by Raffay. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_final_wav2vec2_urdu_asr_project_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_final_wav2vec2_urdu_asr_project_en_4.2.0_3.0_1664020194849.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_final_wav2vec2_urdu_asr_project_en_4.2.0_3.0_1664020194849.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_final_wav2vec2_urdu_asr_project', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_final_wav2vec2_urdu_asr_project", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_final_wav2vec2_urdu_asr_project| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.3 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Voice of the Patients author: John Snow Labs name: ner_vop_slim_wip date: 2023-02-25 tags: [ner, clinical, en, licensed, vop, voice, patient] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true recommended: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient's own sentences. Note: 'wip' suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `AdmissionDischarge`, `Age`, `BodyPart`, `ClinicalDept`, `DateTime`, `Disease`, `Dosage_Strength`, `Drug`, `Duration`, `Employment`, `Form`, `Frequency`, `Gender`, `Laterality`, `Procedure`, `PsychologicalCondition`, `RelationshipStatus`, `Route`, `Symptom`, `Test`, `Vaccine`, `VitalTest` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/VOICE_OF_THE_PATIENTS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_slim_wip_en_4.3.1_3.0_1677342424243.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_slim_wip_en_4.3.1_3.0_1677342424243.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_vop_slim_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = ["Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you."] data = spark.createDataFrame(sample_texts, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_vop_slim_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +--------------------+-----+----+----------------------+ |chunk |begin|end |ner_label | +--------------------+-----+----+----------------------+ |20 year old |10 |20 |Age | |girl |22 |25 |Gender | |hyperthyroid |47 |58 |Disease | |1 month ago |60 |70 |DateTime | |weak |87 |90 |Symptom | |panic attacks |122 |134 |PsychologicalCondition| |depression |137 |146 |PsychologicalCondition| |left |149 |152 |Laterality | |chest |154 |158 |BodyPart | |pain |160 |163 |Symptom | |heart rate |176 |185 |VitalTest | |weight loss |196 |206 |Symptom | |4 months |215 |222 |Duration | |hospital |258 |265 |ClinicalDept | |discharged |276 |285 |AdmissionDischarge | |hospital |292 |299 |ClinicalDept | |blood tests |319 |329 |Test | |brain |332 |336 |BodyPart | |mri |338 |340 |Test | |ultrasound scan |343 |357 |Test | |endoscopy |360 |368 |Procedure | |doctors |391 |397 |Employment | |homeopathy doctor |486 |502 |Employment | |he |512 |513 |Gender | |hyperthyroid |546 |557 |Disease | |TSH |566 |568 |Test | |T3 |579 |580 |Test | |T4 |586 |587 |Test | |b12 deficiency |613 |626 |Disease | |vitamin D deficiency|632 |651 |Disease | |weekly |667 |672 |Frequency | |supplement |674 |683 |Drug | |vitamin D |688 |696 |Drug | |1000 mcg |702 |709 |Dosage_Strength | |b12 |711 |713 |Drug | |daily |715 |719 |Frequency | |homeopathy medicine |733 |751 |Drug | |40 days |757 |763 |Duration | |after 30 days |783 |795 |DateTime | |TSH |801 |803 |Test | |now |812 |814 |DateTime | |weakness |849 |856 |Symptom | |depression |862 |871 |PsychologicalCondition| |last week |912 |920 |DateTime | |rapid heartrate |960 |974 |Symptom | |thyroid |1074 |1080|BodyPart | +--------------------+-----+----+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_slim_wip| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.4 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Route 25.0 4.0 15.0 40.0 0.8621 0.625 0.7246 Procedure 161.0 48.0 70.0 231.0 0.7703 0.697 0.7318 Vaccine 22.0 7.0 8.0 30.0 0.7586 0.7333 0.7458 RelationshipStatus 6.0 2.0 2.0 8.0 0.75 0.75 0.75 Disease 884.0 201.0 285.0 1169.0 0.8147 0.7562 0.7844 Frequency 342.0 61.0 113.0 455.0 0.8486 0.7516 0.7972 Duration 720.0 188.0 146.0 866.0 0.793 0.8314 0.8117 Test 478.0 106.0 103.0 581.0 0.8185 0.8227 0.8206 Symptom 1569.0 337.0 340.0 1909.0 0.8232 0.8219 0.8225 DateTime 1558.0 277.0 296.0 1854.0 0.849 0.8403 0.8447 ClinicalDept 157.0 9.0 48.0 205.0 0.9458 0.7659 0.8464 Form 110.0 28.0 11.0 121.0 0.7971 0.9091 0.8494 Dosage_Strength 184.0 25.0 33.0 217.0 0.8804 0.8479 0.8638 Drug 672.0 109.0 103.0 775.0 0.8604 0.8671 0.8638 VitalTest 73.0 7.0 16.0 89.0 0.9125 0.8202 0.8639 Laterality 262.0 43.0 38.0 300.0 0.859 0.8733 0.8661 Age 236.0 42.0 14.0 250.0 0.8489 0.944 0.8939 PsychologicalCond... 144.0 20.0 14.0 158.0 0.878 0.9114 0.8944 BodyPart 1319.0 139.0 160.0 1479.0 0.9047 0.8918 0.8982 Employment 541.0 25.0 77.0 618.0 0.9558 0.8754 0.9139 AdmissionDischarge 13.0 0.0 1.0 14.0 1.0 0.9286 0.963 Gender 548.0 26.0 12.0 560.0 0.9547 0.9786 0.9665 ``` --- layout: model title: Legal Cooperation Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_cooperation_bert date: 2023-03-05 tags: [en, legal, classification, clauses, cooperation, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Cooperation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Cooperation`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cooperation_bert_en_1.0.0_3.0_1678050678952.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cooperation_bert_en_1.0.0_3.0_1678050678952.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_cooperation_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Cooperation]| |[Other]| |[Other]| |[Cooperation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cooperation_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Cooperation 0.83 0.92 0.87 52 Other 0.94 0.86 0.90 74 accuracy - - 0.89 126 macro-avg 0.88 0.89 0.89 126 weighted-avg 0.89 0.89 0.89 126 ``` --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes (sbertresolve_icd10cm_slim_billable_hcc) author: John Snow Labs name: sbertresolve_icd10cm_slim_billable_hcc date: 2023-05-31 tags: [icd10cm, licensed, slim, en, clinical] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to ICD-10-CM codes using sentence bert embeddings. In this model, synonyms having low cosine similarity to unnormalized terms are dropped. It also returns the official resolution text within the brackets inside the metadata. The model is augmented with synonyms, and previous augmentations are flexed according to cosine distances to unnormalized terms (ground truths). Outputs 7-digit billable ICD codes. In the result, look for aux_label parameter in the metadata to get Hierarchical Condition Categories (HCC) status. This column can be divided to get further details: `billable status || hcc status || hcc score`. For example, if `all_k_aux_labels` is like `1||1||19` which means the `billable status` is 1, `hcc status` is 1, and `hcc score` is 19. ## Predicted Entities {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_slim_billable_hcc_en_4.4.2_3.0_1685498851777.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_slim_billable_hcc_en_4.4.2_3.0_1685498851777.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | ner_chunk| entity|icd10_code| resolutions| all_codes| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.4|[gestational diabetes mellitus [gestational diabetes mellitus], gestatio...|[O24.4, O24.43, P70.2, O24.434, O24.430, O24.435, O24.41, E13, O24.13, O...| |subsequent type two diabetes mellitus|PROBLEM| E11|[type 2 diabetes mellitus [type 2 diabetes mellitus], type 1 diabetes me...|[E11, E10, E11.3, E11.65, E11.1, E10.3, E11.64, E11.0, E11.5, E10.59, E1...| | obesity|PROBLEM| E66.1|[drug-induced obesity [drug-induced obesity], obesity due to excess calo...|[E66.1, E66.0, E66.9, O99.210, O99.213, O99.212, E66, O99.211, E66.8, E6...| | a body mass index|PROBLEM| Z68|[body mass index [bmi] [body mass index [bmi]], body mass index [bmi] 70...|[Z68, Z68.45, Z68.4, Z68.1, Z68.2, Z68.22, Z68.21, Z68.25, Z68.43, Z68.2...| | polyuria|PROBLEM| R35|[polyuria [polyuria], biliuria [biliuria], chyluria [chyluria], anuria a...|[R35, R82.2, R82.0, R34, R80, R35.89, R35.8, R82.991, R82.4, D75.1, L68....| | polydipsia|PROBLEM| R63.1|[polydipsia [polydipsia], polymyositis [polymyositis], polyhydramnios [p...|[R63.1, M33.2, O40, F63.3, Q69, O15, K22.81, N89.7, M30.0, N47.1, D72.82...| | vomiting|PROBLEM| R11.1|[vomiting [vomiting], vomiting of newborn [vomiting of newborn], nausea ...|[R11.1, P92.0, R11, R11.12, G43.A, R11.0, R11.13, R11.14, P92.01, O21, O...| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[bert_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|297.8 MB| |Case sensitive:|false| --- layout: model title: Legal Prices Document Classifier (EURLEX) author: John Snow Labs name: legclf_prices_bert date: 2023-03-06 tags: [en, legal, classification, clauses, prices, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_prices_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Prices or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Prices`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_prices_bert_en_1.0.0_3.0_1678111798081.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_prices_bert_en_1.0.0_3.0_1678111798081.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_prices_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Prices]| |[Other]| |[Other]| |[Prices]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_prices_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.92 0.91 0.92 671 Prices 0.92 0.93 0.92 724 accuracy - - 0.92 1395 macro-avg 0.92 0.92 0.92 1395 weighted-avg 0.92 0.92 0.92 1395 ``` --- layout: model title: Pipeline to Detect clinical events author: John Snow Labs name: ner_events_healthcare_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_events_healthcare](https://nlp.johnsnowlabs.com/2021/04/01/ner_events_healthcare_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_pipeline_en_4.3.0_3.2_1678837044873.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_healthcare_pipeline_en_4.3.0_3.2_1678837044873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_events_healthcare_pipeline", "en", "clinical/models") text = '''The patient presented to the emergency room last evening.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_events_healthcare_pipeline", "en", "clinical/models") val text = "The patient presented to the emergency room last evening." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.healthcare_events.pipeline").predict("""The patient presented to the emergency room last evening.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------------|--------:|------:|:--------------|-------------:| | 0 | presented | 12 | 20 | EVIDENTIAL | 0.6769 | | 1 | the emergency room | 25 | 42 | CLINICAL_DEPT | 0.835967 | | 2 | last evening | 44 | 55 | DATE | 0.59135 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Financial Relation Extraction (Alias) author: John Snow Labs name: finre_org_prod_alias date: 2022-08-17 tags: [en, finance, re, relations, licensed] task: Relation Extraction language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to extract Aliases of Companies or Product names. An "Alias" is a named used in a document to refer to the original name of a company or product. Examples: - John Snow Labs, also known as JSL - John Snow Labs ("JSL") - etc ## Predicted Entities `has_alias`, `has_collective_alias` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finre_org_prod_alias_en_1.0.0_3.2_1660739200247.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finre_org_prod_alias_en_1.0.0_3.2_1660739200247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained("finner_orgs_prods_alias", "en", "finance/models")\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") reDL = finance.RelationExtractionDLModel()\ .pretrained("finre_org_prod_alias", "en", "finance/models")\ .setPredictionThreshold(0.1)\ .setInputCols(["ner_chunk", "document"])\ .setOutputCol("relations") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner_model, ner_converter, reDL]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text=''' On March 12, 2020 we closed a Loan and Security Agreement with Hitachi Capital America Corp. ("Hitachi") the terms of which are described in this report which replaced our credit facility with Western Alliance Bank. ''' lmodel = LightPipeline(model) lmodel.fullAnnotate(text) ```

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence has_alias ORG 64 92 Hitachi Capital America Corp. ALIAS 96 102 Hitachi 0.9983972 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finre_org_prod_alias| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|409.9 MB| ## References Manual annotations on CUAD dataset and 10K filings ## Benchmarking ```bash label Recall Precision F1 Support has_alias 0.920 1.000 0.958 50 has_collective_alias 1.000 0.750 0.857 6 no_rel 1.000 0.957 0.978 44 Avg. 0.973 0.902 0.931 - Weighted-Avg. 0.960 0.966 0.961 - ``` --- layout: model title: Translate Chinyanja to English Pipeline author: John Snow Labs name: translate_ny_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ny, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ny` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ny_en_xx_2.7.0_2.4_1609699579874.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ny_en_xx_2.7.0_2.4_1609699579874.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ny_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ny_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ny.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ny_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Maintenance Of Insurance Clause Binary Classifier author: John Snow Labs name: legclf_maintenance_of_insurance_clause date: 2023-01-29 tags: [en, legal, classification, maintenance, insurance, clauses, maintenance_of_insurance, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `maintenance-of-insurance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `maintenance-of-insurance`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_maintenance_of_insurance_clause_en_1.0.0_3.0_1675004526424.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_maintenance_of_insurance_clause_en_1.0.0_3.0_1675004526424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_maintenance_of_insurance_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[maintenance-of-insurance]| |[other]| |[other]| |[maintenance-of-insurance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_maintenance_of_insurance_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support maintenance-of-insurance 0.98 0.98 0.98 56 other 0.99 0.99 0.99 106 accuracy - - 0.99 162 macro-avg 0.99 0.99 0.99 162 weighted-avg 0.99 0.99 0.99 162 ``` --- layout: model title: Multilingual XLMRoBerta Embeddings (from hfl) author: John Snow Labs name: xlmroberta_embeddings_cino_large date: 2022-05-14 tags: [zh, ko, xlm_roberta, embeddings, xx, open_source] task: Embeddings language: xx edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cino-large` is a Multilingual model orginally trained by `hfl`. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_large_xx_3.4.4_3.0_1652531331865.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_large_xx_3.4.4_3.0_1652531331865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_cino_large| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|2.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/cino-large - https://github.com/ymcui/Chinese-Minority-PLM - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology --- layout: model title: Legal National Accounts Document Classifier (EURLEX) author: John Snow Labs name: legclf_national_accounts_bert date: 2023-03-06 tags: [en, legal, classification, clauses, national_accounts, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_national_accounts_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class National_Accounts or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `National_Accounts`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_national_accounts_bert_en_1.0.0_3.0_1678111671494.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_national_accounts_bert_en_1.0.0_3.0_1678111671494.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_national_accounts_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[National_Accounts]| |[Other]| |[Other]| |[National_Accounts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_national_accounts_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support National_Accounts 0.96 1.00 0.98 27 Other 1.00 0.95 0.97 19 accuracy - - 0.98 46 macro-avg 0.98 0.97 0.98 46 weighted-avg 0.98 0.98 0.98 46 ``` --- layout: model title: English asr_vakyansh_wav2vec2_indian_english_enm_700 TFWav2Vec2ForCTC from Harveenchadha author: John Snow Labs name: asr_vakyansh_wav2vec2_indian_english_enm_700 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_vakyansh_wav2vec2_indian_english_enm_700` is a English model originally trained by Harveenchadha. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_vakyansh_wav2vec2_indian_english_enm_700_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_vakyansh_wav2vec2_indian_english_enm_700_en_4.2.0_3.0_1664094785788.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_vakyansh_wav2vec2_indian_english_enm_700_en_4.2.0_3.0_1664094785788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_vakyansh_wav2vec2_indian_english_enm_700", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_vakyansh_wav2vec2_indian_english_enm_700", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_vakyansh_wav2vec2_indian_english_enm_700| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.6 MB| --- layout: model title: Diseases and Syndromes to UMLS Code Pipeline author: John Snow Labs name: umls_disease_syndrome_resolver_pipeline date: 2022-07-26 tags: [en, licensed, umls, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities (Diseases and Syndromes) with their corresponding UMLS CUI codes. You’ll just feed your text and it will return the corresponding UMLS codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_disease_syndrome_resolver_pipeline_en_4.0.0_3.0_1658825573136.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_disease_syndrome_resolver_pipeline_en_4.0.0_3.0_1658825573136.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("umls_disease_syndrome_resolver_pipeline", "en", "clinical/models") pipeline.annotate("A 34-year-old female with a history of poor appetite, gestational diabetes mellitus, acyclovir allergy and polyuria") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= PretrainedPipeline("umls_disease_syndrome_resolver_pipeline", "en", "clinical/models") val pipeline.annotate("A 34-year-old female with a history of poor appetite, gestational diabetes mellitus, acyclovir allergy and polyuria") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.umls_disease_syndrome_resolver").predict("""A 34-year-old female with a history of poor appetite, gestational diabetes mellitus, acyclovir allergy and polyuria""") ```

## Results ```bash +-----------------------------+---------+---------+ |chunk |ner_label|umls_code| +-----------------------------+---------+---------+ |poor appetite |PROBLEM |C0003123 | |gestational diabetes mellitus|PROBLEM |C0085207 | |acyclovir allergy |PROBLEM |C0571297 | |polyuria |PROBLEM |C0018965 | +-----------------------------+---------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_disease_syndrome_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.4 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: Pipeline to Extract Anatomical Entities from Oncology Texts author: John Snow Labs name: ner_oncology_anatomy_general_pipeline date: 2023-03-08 tags: [licensed, clinical, en, oncology, anatomy, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_anatomy_general](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_anatomy_general_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_pipeline_en_4.3.0_3.2_1678282717993.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_pipeline_en_4.3.0_3.2_1678282717993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_anatomy_general_pipeline", "en", "clinical/models") text = '''The patient presented a mass in her left breast, and a possible metastasis in her lungs and in her liver.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_anatomy_general_pipeline", "en", "clinical/models") val text = "The patient presented a mass in her left breast, and a possible metastasis in her lungs and in her liver." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:----------------|-------------:| | 0 | left | 36 | 39 | Direction | 0.9825 | | 1 | breast | 41 | 46 | Anatomical_Site | 0.9005 | | 2 | lungs | 82 | 86 | Anatomical_Site | 0.9735 | | 3 | liver | 99 | 103 | Anatomical_Site | 0.9817 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_anatomy_general_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Sentence Entity Resolver for ICD10-GM author: John Snow Labs name: sbertresolve_icd10gm date: 2021-09-16 tags: [icd10gm, en, clinical, licensed, de] task: Entity Resolution language: de edition: Healthcare NLP 3.2.2 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-GM codes for the German language using `sent_bert_base_cased` (de) embeddings. ## Predicted Entities `ICD10GM Codes` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_GM_DE/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/14.German_Healthcare_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10gm_de_3.2.2_2.4_1631814227170.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10gm_de_3.2.2_2.4_1631814227170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | code | resolutions | all_codes | all_distances | |---:|:--------|:--------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | Dyspnoe | C671 |Dyspnoe, Schlafapnoe, Dysphonie, Frühsyphilis, Hyperzementose, Hypertrichose, Makrostomie, Dystonie, Nokardiose, Lebersklerose, Dyspareunie, Schizophrenie, Skoliose, Dysurie, Diphyllobothriose, Heterophorie, Rektozele, Enophthalmus, Amyloidose, Hyperventilation, Neurasthenie, Sarkoidose, Psoriasis-Arthropathie, Hyperodontie, Enteroptose| [R06.0, G47.3, R49.0, A51, K03.4, L68, Q18.4, G24, A43, K74.1, N94.1, F20, M41, R30.0, B70.0, H50.5, N81.6, H05.4, E85, R06.4, F48.0, D86, L40.5, K00.1, K63.4] | [0.0000, 2.5602, 3.0529, 3.3310, 3.4645, 3.7148, 3.7568, 3.8115, 3.8557, 3.8577, 3.9448, 3.9681, 3.9799, 3.9889, 4.0036, 4.0773, 4.0825, 4.1342, 4.2031, 4.2155, 4.2313, 4.2341, 4.2775, 4.2802, 4.2823] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_icd10gm| |Compatibility:|Healthcare NLP 3.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10_gm2022_de_code]| |Language:|de| |Case sensitive:|false| --- layout: model title: Legal Sub Advisory Agreement Document Binary Classifier (Longformer) author: John Snow Labs name: legclf_sub_advisory_agreement date: 2022-12-18 tags: [en, legal, classification, licensed, document, longformer, sub, advisory, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_sub_advisory_agreement` model is a Longformer Document Classifier used to classify if the document belongs to the class `sub-advisory-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If your document needs to process more than 4096 tokens, you can try the following: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. ## Predicted Entities `sub-advisory-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sub_advisory_agreement_en_1.0.0_3.0_1671393680636.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sub_advisory_agreement_en_1.0.0_3.0_1671393680636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[sub-advisory-agreement]| |[other]| |[other]| |[sub-advisory-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sub_advisory_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 1.00 0.99 202 sub-advisory-agreement 1.00 0.97 0.99 104 accuracy - - 0.99 306 macro-avg 0.99 0.99 0.99 306 weighted-avg 0.99 0.99 0.99 306 ``` --- layout: model title: English RobertaForQuestionAnswering (from saburbutt) author: John Snow Labs name: roberta_qa_roberta_base_tweetqa_model date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta_base_tweetqa_model` is a English model originally trained by `saburbutt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_tweetqa_model_en_4.0.0_3.0_1655739012245.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_tweetqa_model_en_4.0.0_3.0_1655739012245.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_tweetqa_model","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_tweetqa_model","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.trivia.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_tweetqa_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|432.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saburbutt/roberta_base_tweetqa_model --- layout: model title: Finnish BERT Embeddings (Base Cased) author: John Snow Labs name: bert_finnish_cased date: 2020-08-31 task: Embeddings language: fi edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, fi] supported: true deprecated: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A version of Google's BERT deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. `FinBERT` features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words. `FinBERT` has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train `FinBERT`. These features allow `FinBERT` to outperform not only Multilingual BERT but also all previously proposed models when fine-tuned for Finnish natural language processing tasks. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_finnish_cased_fi_2.6.0_2.4_1598896927571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_finnish_cased_fi_2.6.0_2.4_1598896927571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_finnish_cased", "fi") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Rakastan NLP: tä']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_finnish_cased", "fi") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("Rakastan NLP: tä").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Rakastan NLP: tä"] embeddings_df = nlu.load('fi.embed.bert.cased.').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash fi_embed_bert_cased__embeddings token [0.09888151288032532, -0.72500079870224, 1.001... Rakastan [0.46280959248542786, -0.7008669972419739, 0.9... NLP [0.061913054436445236, 1.1024340391159058, 0.9... : [1.0134484767913818, -0.822027325630188, 1.353... tä ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_finnish_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[fi]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from https://github.com/TurkuNLP/FinBERT --- layout: model title: Arabic BertForQuestionAnswering model (from gfdgdfgdg) author: John Snow Labs name: bert_qa_arap_qa_bert date: 2022-06-02 tags: [ar, open_source, question_answering, bert] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `arap_qa_bert` is a Arabic model orginally trained by `gfdgdfgdg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_arap_qa_bert_ar_4.0.0_3.0_1654179127284.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_arap_qa_bert_ar_4.0.0_3.0_1654179127284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_arap_qa_bert","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_arap_qa_bert","ar") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_arap_qa_bert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ar| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/gfdgdfgdg/arap_qa_bert --- layout: model title: Stopwords Remover for Estonian language (35 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, et, open_source] task: Stop Words Removal language: et edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_et_3.4.1_3.0_1646672955769.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_et_3.4.1_3.0_1646672955769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------+ |result | +------------------+ |[ole, parem, mina]| +------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|et| |Size:|1.4 KB| --- layout: model title: English T5ForConditionalGeneration Cased model (from dsivakumar) author: John Snow Labs name: t5_text2sql date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `text2sql` is a English model originally trained by `dsivakumar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_text2sql_en_4.3.0_3.0_1675157342237.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_text2sql_en_4.3.0_3.0_1675157342237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_text2sql","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_text2sql","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_text2sql| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|283.3 MB| ## References - https://huggingface.co/dsivakumar/text2sql --- layout: model title: English asr_wav2vec2_base_timit_moaiz_exp1 TFWav2Vec2ForCTC from moaiz237 author: John Snow Labs name: asr_wav2vec2_base_timit_moaiz_exp1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_moaiz_exp1` is a English model originally trained by moaiz237. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_moaiz_exp1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_moaiz_exp1_en_4.2.0_3.0_1664036350853.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_moaiz_exp1_en_4.2.0_3.0_1664036350853.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_moaiz_exp1", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_moaiz_exp1", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_moaiz_exp1| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Pipeline to Detect PHI for Deidentification (ner_deidentification_dl) author: John Snow Labs name: ner_deidentify_dl_pipeline date: 2023-03-13 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deidentify_dl](https://nlp.johnsnowlabs.com/2021/03/31/ner_deidentify_dl_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_pipeline_en_4.3.0_3.2_1678735442586.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_pipeline_en_4.3.0_3.2_1678735442586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deidentify_dl_pipeline", "en", "clinical/models") text = '''Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deidentify_dl_pipeline", "en", "clinical/models") val text = "Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deidentify.pipeline").predict("""Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:------------------------------|--------:|------:|:--------------|-------------:| | 0 | 2093-01-13 | 14 | 23 | DATE | 0.9999 | | 1 | David Hale | 27 | 36 | DOCTOR | 0.9913 | | 2 | Hendrickson Ora | 55 | 69 | DOCTOR | 0.95565 | | 3 | 7194334 | 78 | 84 | MEDICALRECORD | 0.9857 | | 4 | 01/13/93 | 93 | 100 | DATE | 0.999 | | 5 | Oliveira | 110 | 117 | DOCTOR | 0.9966 | | 6 | 25 | 121 | 122 | AGE | 0.6433 | | 7 | 2079-11-09 | 150 | 159 | DATE | 0.9984 | | 8 | Cocke County Baptist Hospital | 163 | 191 | HOSPITAL | 0.9466 | | 9 | Keats Street | 200 | 211 | STREET | 0.91485 | | 10 | 302-786-5227 | 221 | 232 | PHONE | 0.7415 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deidentify_dl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from arshiya20) author: John Snow Labs name: distilbert_qa_arshiya20_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `arshiya20`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_arshiya20_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769918439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_arshiya20_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769918439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_arshiya20_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_arshiya20_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_arshiya20_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/arshiya20/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate Mon-Khmer languages to English Pipeline author: John Snow Labs name: translate_mkh_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, mkh, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `mkh` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_mkh_en_xx_2.7.0_2.4_1609698555980.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_mkh_en_xx_2.7.0_2.4_1609698555980.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_mkh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_mkh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.mkh.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_mkh_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English ElectraForQuestionAnswering model (from ahotrod) author: John Snow Labs name: electra_qa_large_discriminator_squad2_512 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra_large_discriminator_squad2_512` is a English model originally trained by `ahotrod`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_large_discriminator_squad2_512_en_4.0.0_3.0_1655921513131.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_large_discriminator_squad2_512_en_4.0.0_3.0_1655921513131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_large_discriminator_squad2_512","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_large_discriminator_squad2_512","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.electra.large_512d").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_large_discriminator_squad2_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ahotrod/electra_large_discriminator_squad2_512 --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from ZYW) author: John Snow Labs name: distilbert_qa_model_zyw date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `en-de-model` is a Multilingual model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_model_zyw_xx_4.3.0_3.0_1672774991529.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_model_zyw_xx_4.3.0_3.0_1672774991529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_model_zyw","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_model_zyw","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_model_zyw| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/en-de-model --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_large_uncased_squadv2 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-squadv2` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_squadv2_en_4.0.0_3.0_1654536822607.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_squadv2_en_4.0.0_3.0_1654536822607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_uncased_v2.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-large-uncased-squadv2 - https://arxiv.org/pdf/1810.04805v2.pdf%5D --- layout: model title: English DistilBertForQuestionAnswering model (from mvonwyl) author: John Snow Labs name: distilbert_qa_mvonwyl_base_uncased_finetuned_squad2 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad2` is a English model originally trained by `mvonwyl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_mvonwyl_base_uncased_finetuned_squad2_en_4.0.0_3.0_1654726852807.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_mvonwyl_base_uncased_finetuned_squad2_en_4.0.0_3.0_1654726852807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mvonwyl_base_uncased_finetuned_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mvonwyl_base_uncased_finetuned_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_mvonwyl_base_uncased_finetuned_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/mvonwyl/distilbert-base-uncased-finetuned-squad2 --- layout: model title: Context Spell Checker Pipeline for English author: John Snow Labs name: spellcheck_dl_pipeline date: 2022-04-14 tags: [spellcheck, spelling_corrector, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained spellchecker pipeline is built on the top of [spellcheck_dl](https://nlp.johnsnowlabs.com/2022/03/28/spellcheck_dl_en_3_0.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.1_3.0_1649937383379.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.1_3.0_1649937383379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_finetuned_cuad","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_finetuned_cuad","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_finetuned_cuad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|913.5 MB| ## References - https://huggingface.co/mrm8488/T5-base-finetuned-cuad --- layout: model title: English ElectraForQuestionAnswering model (from valhalla) author: John Snow Labs name: electra_qa_base_discriminator_finetuned_squadv1 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-discriminator-finetuned_squadv1` is a English model originally trained by `valhalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv1_en_4.0.0_3.0_1655920510384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv1_en_4.0.0_3.0_1655920510384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.base.by_valhalla").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_discriminator_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/valhalla/electra-base-discriminator-finetuned_squadv1 - https://github.com/patil-suraj/ --- layout: model title: Clinical Deidentification (Spanish) author: John Snow Labs name: clinical_deidentification date: 2022-02-14 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with sciwiki_300d embeddings and can be used to deidentify PHI information from medical texts in Spanish. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask, fake or obfuscate the following entities: `AGE`, `DATE`, `PROFESSION`, `E-MAIL`, `USERNAME`, `LOCATION`, `DOCTOR`, `HOSPITAL`, `PATIENT`, `URL`, `IP`, `MEDICALRECORD`, `IDNUM`, `ORGANIZATION`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `PLATE`, `SEX` and `IPADDR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.3.4_3.0_1644832415526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_es_3.3.4_3.0_1644832415526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "es", "clinical/models") sample = """Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com """ result = deid_pipeline .annotate(sample) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification", "es", "clinical/models") sample = "Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("es.deid.clinical").predict("""Datos del paciente. Nombre: Jose . Apellidos: Aranda Martinez. NHC: 2748903. NASS: 26 37482910 04. Domicilio: Calle Losada Martí 23, 5 B.. Localidad/ Provincia: Madrid. CP: 28016. Datos asistenciales. Fecha de nacimiento: 15/04/1977. País: España. Edad: 37 años Sexo: F. Fecha de Ingreso: 05/06/2018. Médico: María Merino Viveros NºCol: 28 28 35489. Informe clínico del paciente: varón de 37 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Extremadura en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Rosa de Bengala +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente mejora significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. María Merino Viveros Hospital Universitario de Getafe Servicio de Endocrinología y Nutrición Carretera de Toledo km 12,500 28905 Getafe - Madrid (España) Correo electrónico: marietta84@hotmail.com """) ```

## Results ```bash Masked with entity labels ------------------------------ Datos del paciente. Nombre: . Apellidos: . NHC: . NASS: 04. Domicilio: , 5 B.. Localidad/ Provincia: . CP: . Datos asistenciales. Fecha de nacimiento: . País: . Edad: años Sexo: . Fecha de Ingreso: . : María Merino Viveros NºCol: . Informe clínico del paciente: de años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Servicio de Endocrinología y Nutrición km 12,500 28905 - () Correo electrónico: Masked with chars ------------------------------ Datos del paciente. Nombre: [**] . Apellidos: [*************]. NHC: [*****]. NASS: ** [******] 04. Domicilio: [*******************], 5 B.. Localidad/ Provincia: [****]. CP: [***]. Datos asistenciales. Fecha de nacimiento: [********]. País: [****]. Edad: ** años Sexo: *. Fecha de Ingreso: [********]. [****]: María Merino Viveros NºCol: ** ** [***]. Informe clínico del paciente: [***] de ** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en [*********] en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: [*************] +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente [****] significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. [******************] [******************************] Servicio de Endocrinología y Nutrición [*****************] km 12,500 28905 [****] - [****] ([****]) Correo electrónico: [********************] Masked with fixed length chars ------------------------------ Datos del paciente. Nombre: **** . Apellidos: ****. NHC: ****. NASS: **** **** 04. Domicilio: ****, 5 B.. Localidad/ Provincia: ****. CP: ****. Datos asistenciales. Fecha de nacimiento: ****. País: ****. Edad: **** años Sexo: ****. Fecha de Ingreso: ****. ****: María Merino Viveros NºCol: **** **** ****. Informe clínico del paciente: **** de **** años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en **** en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: **** +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente **** significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. **** **** Servicio de Endocrinología y Nutrición **** km 12,500 28905 **** - **** (****) Correo electrónico: **** Obfuscated ------------------------------ Datos del paciente. Nombre: Sr. Lerma . Apellidos: Aristides Gonzalez Gelabert. NHC: BBBBBBBBQR648597. NASS: 041010000011 RZRM020101906017 04. Domicilio: Valencia, 5 B.. Localidad/ Provincia: Madrid. CP: 99335. Datos asistenciales. Fecha de nacimiento: 25/04/1977. País: Barcelona. Edad: 8 años Sexo: F.. Fecha de Ingreso: 02/08/2018. transportista: María Merino Viveros NºCol: olegario10 olegario10 felisa78. Informe clínico del paciente: RZRM020101906017 de 8 años con vida previa activa que refiere dolores osteoarticulares de localización variable en el último mes y fiebre en la última semana con picos (matutino y vespertino) de 40 C las últimas 24-48 horas, por lo que acude al Servicio de Urgencias. Antes de comenzar el cuadro estuvo en Madrid en una región endémica de brucella, ingiriendo leche de cabra sin pasteurizar y queso de dicho ganado. Entre los comensales aparecieron varios casos de brucelosis. Durante el ingreso para estudio del síndrome febril con antecedentes epidemiológicos de posible exposición a Brucella presenta un cuadro de orquiepididimitis derecha. La exploración física revela: Tª 40,2 C; T.A: 109/68 mmHg; Fc: 105 lpm. Se encuentra consciente, orientado, sudoroso, eupneico, con buen estado de nutrición e hidratación. En cabeza y cuello no se palpan adenopatías, ni bocio ni ingurgitación de vena yugular, con pulsos carotídeos simétricos. Auscultación cardíaca rítmica, sin soplos, roces ni extratonos. Auscultación pulmonar con conservación del murmullo vesicular. Abdomen blando, depresible, sin masas ni megalias. En la exploración neurológica no se detectan signos meníngeos ni datos de focalidad. Extremidades sin varices ni edemas. Pulsos periféricos presentes y simétricos. En la exploración urológica se aprecia el teste derecho aumentado de tamaño, no adherido a piel, con zonas de fluctuación e intensamente doloroso a la palpación, con pérdida del límite epidídimo-testicular y transiluminación positiva. Los datos analíticos muestran los siguentes resultados: Hemograma: Hb 13,7 g/dl; leucocitos 14.610/mm3 (neutrófilos 77%); plaquetas 206.000/ mm3. VSG: 40 mm 1ª hora. Coagulación: TQ 87%; TTPA 25,8 seg. Bioquímica: Glucosa 117 mg/dl; urea 29 mg/dl; creatinina 0,9 mg/dl; sodio 136 mEq/l; potasio 3,6 mEq/l; GOT 11 U/l; GPT 24 U/l; GGT 34 U/l; fosfatasa alcalina 136 U/l; calcio 8,3 mg/dl. Orina: sedimento normal. Durante el ingreso se solicitan Hemocultivos: positivo para Brucella y Serologías específicas para Brucella: Dra. Laguna +++; Test de Coombs > 1/1280; Brucellacapt > 1/5120. Las pruebas de imagen solicitadas ( Rx tórax, Ecografía abdominal, TAC craneal, Ecocardiograma transtorácico) no evidencian patología significativa, excepto la Ecografía testicular, que muestra engrosamiento de la bolsa escrotal con pequeña cantidad de líquido con septos y testículo aumentado de tamaño con pequeñas zonas hipoecoicas en su interior que pueden representar microabscesos. Con el diagnóstico de orquiepididimitis secundaria a Brucella se instaura tratamiento sintomático (antitérmicos, antiinflamatorios, reposo y elevación testicular) así como tratamiento antibiótico específico: Doxiciclina 100 mg vía oral cada 12 horas (durante 6 semanas) y Estreptomicina 1 gramo intramuscular cada 24 horas (durante 3 semanas). El paciente 041010000011 significativamente de su cuadro tras una semana de ingreso, decidiéndose el alta a su domicilio donde completó la pauta de tratamiento antibiótico. En revisiones sucesivas en consultas se constató la completa remisión del cuadro. Remitido por: Dra. Reinaldo Manjón Malo Barcelona Servicio de Endocrinología y Nutrición Valencia km 12,500 28905 Bilbao - Madrid (Barcelona) Correo electrónico: quintanasalome@example.net ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|281.3 MB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetectorDLModel - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - nlp.NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: Pipeline to Detect Chemical Compounds and Genes author: John Snow Labs name: bert_token_classifier_ner_chemprot_pipeline date: 2022-03-15 tags: [bert_token_classifier, ner, chemprot, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemprot](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemprot_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMPROT_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CHEMPROT_CLINICAL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_2.4_1647341872538.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_2.4_1647341872538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python chemprot_pipeline = PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") chemprot_pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` ```scala val chemprot_pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") chemprot_pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemprot_pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_yashwantk_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_yashwantk_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_yashwantk_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/yashwantk/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Non solicitation Clause Binary Classifier author: John Snow Labs name: legclf_non_solicitation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `non-solicitation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `non-solicitation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_non_solicitation_clause_en_1.0.0_3.2_1660122751584.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_non_solicitation_clause_en_1.0.0_3.2_1660122751584.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_non_solicitation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[non-solicitation]| |[other]| |[other]| |[non-solicitation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_non_solicitation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support non-solicitation 1.00 0.95 0.98 22 other 0.99 1.00 0.99 84 accuracy - - 0.99 106 macro-avg 0.99 0.98 0.99 106 weighted-avg 0.99 0.99 0.99 106 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab4 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab4 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab4` is a English model originally trained by sameearif88. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab4_en_4.2.0_3.0_1664019147552.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab4_en_4.2.0_3.0_1664019147552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab4', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab4", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab4| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering Cased model (from clementgyj) author: John Snow Labs name: bert_qa_clementgyj_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `clementgyj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_clementgyj_finetuned_squad_en_4.0.0_3.0_1657186454797.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_clementgyj_finetuned_squad_en_4.0.0_3.0_1657186454797.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_clementgyj_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_clementgyj_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_clementgyj_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/clementgyj/bert-finetuned-squad --- layout: model title: Translate English to Welsh Pipeline author: John Snow Labs name: translate_en_cy date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, cy, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `cy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_cy_xx_2.7.0_2.4_1609698924389.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_cy_xx_2.7.0_2.4_1609698924389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_cy", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_cy", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.cy').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_cy| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_deit_small_patch16_224 ViTForImageClassification from facebook author: John Snow Labs name: image_classifier_vit_deit_small_patch16_224 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_deit_small_patch16_224` is a English model originally trained by facebook. ## Predicted Entities `turnstile`, `damselfly`, `mixing bowl`, `sea snake`, `cockroach, roach`, `buckle`, `beer glass`, `bulbul`, `lumbermill, sawmill`, `whippet`, `Australian terrier`, `television, television system`, `hoopskirt, crinoline`, `horse cart, horse-cart`, `guillotine`, `malamute, malemute, Alaskan malamute`, `coyote, prairie wolf, brush wolf, Canis latrans`, `colobus, colobus monkey`, `hognose snake, puff adder, sand viper`, `sock`, `burrito`, `printer`, `bathing cap, swimming cap`, `chiton, coat-of-mail shell, sea cradle, polyplacophore`, `Rottweiler`, `cello, violoncello`, `pitcher, ewer`, `computer keyboard, keypad`, `bow`, `peacock`, `ballplayer, baseball player`, `refrigerator, icebox`, `solar dish, solar collector, solar furnace`, `passenger car, coach, carriage`, `African chameleon, Chamaeleo chamaeleon`, `oboe, hautboy, hautbois`, `toyshop`, `Leonberg`, `howler monkey, howler`, `bluetick`, `African elephant, Loxodonta africana`, `American lobster, Northern lobster, Maine lobster, Homarus americanus`, `combination lock`, `black-and-tan coonhound`, `bonnet, poke bonnet`, `harvester, reaper`, `Appenzeller`, `iron, smoothing iron`, `electric locomotive`, `lycaenid, lycaenid butterfly`, `sandbar, sand bar`, `Cardigan, Cardigan Welsh corgi`, `pencil sharpener`, `jean, blue jean, denim`, `backpack, back pack, knapsack, packsack, rucksack, haversack`, `monitor`, `ice cream, icecream`, `apiary, bee house`, `water jug`, `American coot, marsh hen, mud hen, water hen, Fulica americana`, `ground beetle, carabid beetle`, `jigsaw puzzle`, `ant, emmet, pismire`, `wreck`, `kuvasz`, `gyromitra`, `Ibizan hound, Ibizan Podenco`, `brown bear, bruin, Ursus arctos`, `bolo tie, bolo, bola tie, bola`, `Pembroke, Pembroke Welsh corgi`, `French bulldog`, `prison, prison house`, `ballpoint, ballpoint pen, ballpen, Biro`, `stage`, `airliner`, `dogsled, dog sled, dog sleigh`, `redshank, Tringa totanus`, `menu`, `Indian cobra, Naja naja`, `swab, swob, mop`, `window screen`, `brain coral`, `artichoke, globe artichoke`, `loupe, jeweler's loupe`, `loudspeaker, speaker, speaker unit, loudspeaker system, speaker system`, `panpipe, pandean pipe, syrinx`, `wok`, `croquet ball`, `plate`, `scoreboard`, `Samoyed, Samoyede`, `ocarina, sweet potato`, `beaver`, `borzoi, Russian wolfhound`, `horizontal bar, high bar`, `stretcher`, `seat belt, seatbelt`, `obelisk`, `forklift`, `feather boa, boa`, `frying pan, frypan, skillet`, `barbershop`, `hamper`, `face powder`, `Siamese cat, Siamese`, `ladle`, `dingo, warrigal, warragal, Canis dingo`, `mountain tent`, `head cabbage`, `echidna, spiny anteater, anteater`, `Polaroid camera, Polaroid Land camera`, `dumbbell`, `espresso`, `notebook, notebook computer`, `Norfolk terrier`, `binoculars, field glasses, opera glasses`, `carpenter's kit, tool kit`, `moving van`, `catamaran`, `tiger beetle`, `bikini, two-piece`, `Siberian husky`, `studio couch, day bed`, `bulletproof vest`, `lawn mower, mower`, `promontory, headland, head, foreland`, `soap dispenser`, `vulture`, `dam, dike, dyke`, `brambling, Fringilla montifringilla`, `toilet tissue, toilet paper, bathroom tissue`, `ringlet, ringlet butterfly`, `tiger cat`, `mobile home, manufactured home`, `Norwich terrier`, `little blue heron, Egretta caerulea`, `English setter`, `Tibetan mastiff`, `rocking chair, rocker`, `mask`, `maze, labyrinth`, `bookcase`, `viaduct`, `sweatshirt`, `plow, plough`, `basenji`, `typewriter keyboard`, `Windsor tie`, `coral fungus`, `desktop computer`, `Kerry blue terrier`, `Angora, Angora rabbit`, `can opener, tin opener`, `shield, buckler`, `triumphal arch`, `horned viper, cerastes, sand viper, horned asp, Cerastes cornutus`, `miniature schnauzer`, `tape player`, `jaguar, panther, Panthera onca, Felis onca`, `hook, claw`, `file, file cabinet, filing cabinet`, `chime, bell, gong`, `shower curtain`, `window shade`, `acoustic guitar`, `gas pump, gasoline pump, petrol pump, island dispenser`, `cicada, cicala`, `Petri dish`, `paintbrush`, `banana`, `chickadee`, `mountain bike, all-terrain bike, off-roader`, `lighter, light, igniter, ignitor`, `oil filter`, `cab, hack, taxi, taxicab`, `Christmas stocking`, `rugby ball`, `black widow, Latrodectus mactans`, `bustard`, `fiddler crab`, `web site, website, internet site, site`, `chocolate sauce, chocolate syrup`, `chainlink fence`, `fireboat`, `cocktail shaker`, `airship, dirigible`, `projectile, missile`, `bagel, beigel`, `screwdriver`, `oystercatcher, oyster catcher`, `pot, flowerpot`, `water bottle`, `Loafer`, `drumstick`, `soccer ball`, `cairn, cairn terrier`, `padlock`, `tow truck, tow car, wrecker`, `bloodhound, sleuthhound`, `punching bag, punch bag, punching ball, punchball`, `great grey owl, great gray owl, Strix nebulosa`, `scale, weighing machine`, `trench coat`, `briard`, `cheetah, chetah, Acinonyx jubatus`, `entertainment center`, `Boston bull, Boston terrier`, `Arabian camel, dromedary, Camelus dromedarius`, `steam locomotive`, `coil, spiral, volute, whorl, helix`, `plane, carpenter's plane, woodworking plane`, `gondola`, `spider web, spider's web`, `bathtub, bathing tub, bath, tub`, `pelican`, `miniature poodle`, `cowboy boot`, `perfume, essence`, `lakeside, lakeshore`, `timber wolf, grey wolf, gray wolf, Canis lupus`, `moped`, `sunscreen, sunblock, sun blocker`, `Brabancon griffon`, `puffer, pufferfish, blowfish, globefish`, `lifeboat`, `pool table, billiard table, snooker table`, `Bouvier des Flandres, Bouviers des Flandres`, `Pomeranian`, `theater curtain, theatre curtain`, `marimba, xylophone`, `baboon`, `vacuum, vacuum cleaner`, `pill bottle`, `pick, plectrum, plectron`, `hen`, `American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier`, `digital watch`, `pier`, `oxygen mask`, `Tibetan terrier, chrysanthemum dog`, `ostrich, Struthio camelus`, `water ouzel, dipper`, `drilling platform, offshore rig`, `magnetic compass`, `throne`, `butternut squash`, `minibus`, `EntleBucher`, `carousel, carrousel, merry-go-round, roundabout, whirligig`, `hot pot, hotpot`, `rain barrel`, `wood rabbit, cottontail, cottontail rabbit`, `miniature pinscher`, `partridge`, `three-toed sloth, ai, Bradypus tridactylus`, `English springer, English springer spaniel`, `corkscrew, bottle screw`, `fur coat`, `robin, American robin, Turdus migratorius`, `dowitcher`, `ruddy turnstone, Arenaria interpres`, `water snake`, `stove`, `Great Pyrenees`, `soft-coated wheaten terrier`, `carbonara`, `snail`, `breastplate, aegis, egis`, `wolf spider, hunting spider`, `hatchet`, `CD player`, `axolotl, mud puppy, Ambystoma mexicanum`, `pomegranate`, `poncho`, `leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea`, `lorikeet`, `spatula`, `jay`, `platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus`, `stethoscope`, `flagpole, flagstaff`, `coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch`, `agama`, `red wolf, maned wolf, Canis rufus, Canis niger`, `beaker`, `eft`, `pretzel`, `brassiere, bra, bandeau`, `frilled lizard, Chlamydosaurus kingi`, `joystick`, `goldfish, Carassius auratus`, `fig`, `maypole`, `caldron, cauldron`, `admiral`, `impala, Aepyceros melampus`, `spotted salamander, Ambystoma maculatum`, `syringe`, `hog, pig, grunter, squealer, Sus scrofa`, `handkerchief, hankie, hanky, hankey`, `tarantula`, `cheeseburger`, `pinwheel`, `sax, saxophone`, `dung beetle`, `broccoli`, `cassette player`, `milk can`, `traffic light, traffic signal, stoplight`, `shovel`, `sarong`, `tabby, tabby cat`, `parallel bars, bars`, `ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle`, `quill, quill pen`, `giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca`, `steel drum`, `quail`, `Blenheim spaniel`, `wig`, `hamster`, `ice lolly, lolly, lollipop, popsicle`, `seashore, coast, seacoast, sea-coast`, `chest`, `worm fence, snake fence, snake-rail fence, Virginia fence`, `missile`, `beer bottle`, `yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum`, `breakwater, groin, groyne, mole, bulwark, seawall, jetty`, `white wolf, Arctic wolf, Canis lupus tundrarum`, `guacamole`, `porcupine, hedgehog`, `trolleybus, trolley coach, trackless trolley`, `greenhouse, nursery, glasshouse`, `trimaran`, `Italian greyhound`, `potter's wheel`, `jacamar`, `wallet, billfold, notecase, pocketbook`, `Lakeland terrier`, `green lizard, Lacerta viridis`, `indigo bunting, indigo finch, indigo bird, Passerina cyanea`, `green mamba`, `walking stick, walkingstick, stick insect`, `crossword puzzle, crossword`, `eggnog`, `barrow, garden cart, lawn cart, wheelbarrow`, `remote control, remote`, `bicycle-built-for-two, tandem bicycle, tandem`, `wool, woolen, woollen`, `black grouse`, `abaya`, `marmoset`, `golf ball`, `jeep, landrover`, `Mexican hairless`, `dishwasher, dish washer, dishwashing machine`, `jersey, T-shirt, tee shirt`, `planetarium`, `goose`, `mailbox, letter box`, `capuchin, ringtail, Cebus capucinus`, `marmot`, `orangutan, orang, orangutang, Pongo pygmaeus`, `coffeepot`, `ambulance`, `shopping basket`, `pop bottle, soda bottle`, `red fox, Vulpes vulpes`, `crash helmet`, `street sign`, `affenpinscher, monkey pinscher, monkey dog`, `Arctic fox, white fox, Alopex lagopus`, `sidewinder, horned rattlesnake, Crotalus cerastes`, `ruffed grouse, partridge, Bonasa umbellus`, `muzzle`, `measuring cup`, `canoe`, `reflex camera`, `fox squirrel, eastern fox squirrel, Sciurus niger`, `French loaf`, `killer whale, killer, orca, grampus, sea wolf, Orcinus orca`, `dial telephone, dial phone`, `thimble`, `bubble`, `vizsla, Hungarian pointer`, `running shoe`, `mailbag, postbag`, `radio telescope, radio reflector`, `piggy bank, penny bank`, `Chihuahua`, `chambered nautilus, pearly nautilus, nautilus`, `Airedale, Airedale terrier`, `kimono`, `green snake, grass snake`, `rubber eraser, rubber, pencil eraser`, `upright, upright piano`, `orange`, `revolver, six-gun, six-shooter`, `ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin`, `drum, membranophone, tympan`, `Dungeness crab, Cancer magister`, `lipstick, lip rouge`, `gong, tam-tam`, `fountain`, `tub, vat`, `malinois`, `sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita`, `German short-haired pointer`, `apron`, `Irish setter, red setter`, `dishrag, dishcloth`, `school bus`, `candle, taper, wax light`, `bib`, `cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM`, `power drill`, `English foxhound`, `miniskirt, mini`, `swing`, `slug`, `hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa`, `rifle`, `Saluki, gazelle hound`, `Sealyham terrier, Sealyham`, `bullet train, bullet`, `hyena, hyaena`, `ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus`, `toy terrier`, `goblet`, `safe`, `cup`, `electric guitar`, `red wine`, `restaurant, eating house, eating place, eatery`, `wall clock`, `washbasin, handbasin, washbowl, lavabo, wash-hand basin`, `red-breasted merganser, Mergus serrator`, `crate`, `banded gecko`, `hippopotamus, hippo, river horse, Hippopotamus amphibius`, `tick`, `tripod`, `sombrero`, `desk`, `sea slug, nudibranch`, `racer, race car, racing car`, `pizza, pizza pie`, `dining table, board`, `Saint Bernard, St Bernard`, `komondor`, `electric ray, crampfish, numbfish, torpedo`, `prairie chicken, prairie grouse, prairie fowl`, `coffee mug`, `hammer`, `golfcart, golf cart`, `unicycle, monocycle`, `bison`, `soup bowl`, `rapeseed`, `golden retriever`, `plastic bag`, `grey fox, gray fox, Urocyon cinereoargenteus`, `water tower`, `house finch, linnet, Carpodacus mexicanus`, `barbell`, `hair slide`, `tiger, Panthera tigris`, `black-footed ferret, ferret, Mustela nigripes`, `meat loaf, meatloaf`, `hand blower, blow dryer, blow drier, hair dryer, hair drier`, `overskirt`, `gibbon, Hylobates lar`, `Gila monster, Heloderma suspectum`, `toucan`, `snowmobile`, `pencil box, pencil case`, `scuba diver`, `cloak`, `Sussex spaniel`, `otter`, `Greater Swiss Mountain dog`, `great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias`, `torch`, `magpie`, `tiger shark, Galeocerdo cuvieri`, `wing`, `Border collie`, `bell cote, bell cot`, `sea anemone, anemone`, `teapot`, `sea urchin`, `screen, CRT screen`, `bookshop, bookstore, bookstall`, `oscilloscope, scope, cathode-ray oscilloscope, CRO`, `crib, cot`, `police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria`, `hartebeest`, `manhole cover`, `iPod`, `rock python, rock snake, Python sebae`, `nipple`, `suspension bridge`, `safety pin`, `sea lion`, `cougar, puma, catamount, mountain lion, painter, panther, Felis concolor`, `mantis, mantid`, `wardrobe, closet, press`, `projector`, `Granny Smith`, `diamondback, diamondback rattlesnake, Crotalus adamanteus`, `pirate, pirate ship`, `espresso maker`, `African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus`, `cradle`, `common newt, Triturus vulgaris`, `tricycle, trike, velocipede`, `bobsled, bobsleigh, bob`, `thunder snake, worm snake, Carphophis amoenus`, `thresher, thrasher, threshing machine`, `banjo`, `armadillo`, `pajama, pyjama, pj's, jammies`, `ski`, `Maltese dog, Maltese terrier, Maltese`, `leafhopper`, `book jacket, dust cover, dust jacket, dust wrapper`, `silky terrier, Sydney silky`, `Shih-Tzu`, `wallaby, brush kangaroo`, `cardigan`, `sturgeon`, `freight car`, `home theater, home theatre`, `sundial`, `African crocodile, Nile crocodile, Crocodylus niloticus`, `odometer, hodometer, mileometer, milometer`, `sliding door`, `vine snake`, `West Highland white terrier`, `mongoose`, `hornbill`, `beagle`, `European gallinule, Porphyrio porphyrio`, `submarine, pigboat, sub, U-boat`, `Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis`, `cock`, `pedestal, plinth, footstall`, `accordion, piano accordion, squeeze box`, `gown`, `lynx, catamount`, `guenon, guenon monkey`, `Walker hound, Walker foxhound`, `standard schnauzer`, `reel`, `hip, rose hip, rosehip`, `grasshopper, hopper`, `Dutch oven`, `stone wall`, `hard disc, hard disk, fixed disk`, `snow leopard, ounce, Panthera uncia`, `shopping cart`, `digital clock`, `hourglass`, `Border terrier`, `Old English sheepdog, bobtail`, `academic gown, academic robe, judge's robe`, `spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish`, `spotlight, spot`, `dome`, `barn spider, Araneus cavaticus`, `bee eater`, `basketball`, `cliff dwelling`, `folding chair`, `isopod`, `Doberman, Doberman pinscher`, `bittern`, `sunglasses, dark glasses, shades`, `picket fence, paling`, `Crock Pot`, `ibex, Capra ibex`, `neck brace`, `cardoon`, `cassette`, `amphibian, amphibious vehicle`, `minivan`, `analog clock`, `trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi`, `yurt`, `cliff, drop, drop-off`, `Bernese mountain dog`, `teddy, teddy bear`, `sloth bear, Melursus ursinus, Ursus ursinus`, `bassoon`, `toaster`, `ptarmigan`, `Gordon setter`, `night snake, Hypsiglena torquata`, `grand piano, grand`, `purse`, `clumber, clumber spaniel`, `shoji`, `hair spray`, `maillot`, `knee pad`, `space heater`, `bottlecap`, `chiffonier, commode`, `chain saw, chainsaw`, `sulphur butterfly, sulfur butterfly`, `pay-phone, pay-station`, `kelpie`, `mouse, computer mouse`, `car wheel`, `cornet, horn, trumpet, trump`, `container ship, containership, container vessel`, `matchstick`, `scabbard`, `American black bear, black bear, Ursus americanus, Euarctos americanus`, `langur`, `rock crab, Cancer irroratus`, `lionfish`, `speedboat`, `black stork, Ciconia nigra`, `knot`, `disk brake, disc brake`, `mosquito net`, `white stork, Ciconia ciconia`, `abacus`, `titi, titi monkey`, `grocery store, grocery, food market, market`, `waffle iron`, `pickelhaube`, `wooden spoon`, `Norwegian elkhound, elkhound`, `earthstar`, `sewing machine`, `balance beam, beam`, `potpie`, `chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour`, `Staffordshire bullterrier, Staffordshire bull terrier`, `switch, electric switch, electrical switch`, `dhole, Cuon alpinus`, `paddle, boat paddle`, `limousine, limo`, `Shetland sheepdog, Shetland sheep dog, Shetland`, `space bar`, `library`, `paddlewheel, paddle wheel`, `alligator lizard`, `Band Aid`, `Persian cat`, `bull mastiff`, `tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui`, `sports car, sport car`, `football helmet`, `laptop, laptop computer`, `lens cap, lens cover`, `tennis ball`, `violin, fiddle`, `lab coat, laboratory coat`, `cinema, movie theater, movie theatre, movie house, picture palace`, `weasel`, `bow tie, bow-tie, bowtie`, `macaw`, `dough`, `whiskey jug`, `microphone, mike`, `spoonbill`, `bassinet`, `mud turtle`, `velvet`, `warthog`, `plunger, plumber's helper`, `dugong, Dugong dugon`, `honeycomb`, `badger`, `dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk`, `bee`, `doormat, welcome mat`, `fountain pen`, `giant schnauzer`, `assault rifle, assault gun`, `limpkin, Aramus pictus`, `siamang, Hylobates syndactylus, Symphalangus syndactylus`, `albatross, mollymawk`, `confectionery, confectionary, candy store`, `harp`, `parachute, chute`, `barrel, cask`, `tank, army tank, armored combat vehicle, armoured combat vehicle`, `collie`, `kite`, `puck, hockey puck`, `stupa, tope`, `buckeye, horse chestnut, conker`, `patio, terrace`, `broom`, `Dandie Dinmont, Dandie Dinmont terrier`, `scorpion`, `agaric`, `balloon`, `bucket, pail`, `squirrel monkey, Saimiri sciureus`, `Eskimo dog, husky`, `zebra`, `garter snake, grass snake`, `indri, indris, Indri indri, Indri brevicaudatus`, `tractor`, `guinea pig, Cavia cobaya`, `maraca`, `red-backed sandpiper, dunlin, Erolia alpina`, `bullfrog, Rana catesbeiana`, `trilobite`, `Japanese spaniel`, `gorilla, Gorilla gorilla`, `monastery`, `centipede`, `terrapin`, `llama`, `long-horned beetle, longicorn, longicorn beetle`, `boxer`, `curly-coated retriever`, `mortar`, `hammerhead, hammerhead shark`, `goldfinch, Carduelis carduelis`, `garden spider, Aranea diademata`, `stopwatch, stop watch`, `grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus`, `leaf beetle, chrysomelid`, `birdhouse`, `king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica`, `stole`, `bell pepper`, `radiator`, `flatworm, platyhelminth`, `mushroom`, `Scotch terrier, Scottish terrier, Scottie`, `liner, ocean liner`, `toilet seat`, `lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens`, `zucchini, courgette`, `harvestman, daddy longlegs, Phalangium opilio`, `Newfoundland, Newfoundland dog`, `flamingo`, `whiptail, whiptail lizard`, `geyser`, `cleaver, meat cleaver, chopper`, `sea cucumber, holothurian`, `American egret, great white heron, Egretta albus`, `parking meter`, `beacon, lighthouse, beacon light, pharos`, `coucal`, `motor scooter, scooter`, `mitten`, `cannon`, `weevil`, `megalith, megalithic structure`, `stinkhorn, carrion fungus`, `ear, spike, capitulum`, `box turtle, box tortoise`, `snowplow, snowplough`, `tench, Tinca tinca`, `modem`, `tobacco shop, tobacconist shop, tobacconist`, `barn`, `skunk, polecat, wood pussy`, `African grey, African gray, Psittacus erithacus`, `Madagascar cat, ring-tailed lemur, Lemur catta`, `holster`, `barometer`, `sleeping bag`, `washer, automatic washer, washing machine`, `recreational vehicle, RV, R.V.`, `drake`, `tray`, `butcher shop, meat market`, `china cabinet, china closet`, `medicine chest, medicine cabinet`, `photocopier`, `Yorkshire terrier`, `starfish, sea star`, `racket, racquet`, `park bench`, `Labrador retriever`, `whistle`, `clog, geta, patten, sabot`, `volcano`, `quilt, comforter, comfort, puff`, `leopard, Panthera pardus`, `cauliflower`, `swimming trunks, bathing trunks`, `American chameleon, anole, Anolis carolinensis`, `alp`, `mortarboard`, `barracouta, snoek`, `cocker spaniel, English cocker spaniel, cocker`, `space shuttle`, `beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon`, `harmonica, mouth organ, harp, mouth harp`, `gasmask, respirator, gas helmet`, `wombat`, `Model T`, `wild boar, boar, Sus scrofa`, `hermit crab`, `flat-coated retriever`, `rotisserie`, `jinrikisha, ricksha, rickshaw`, `trifle`, `bannister, banister, balustrade, balusters, handrail`, `go-kart`, `bakery, bakeshop, bakehouse`, `ski mask`, `dock, dockage, docking facility`, `Egyptian cat`, `oxcart`, `redbone`, `shoe shop, shoe-shop, shoe store`, `convertible`, `ox`, `crayfish, crawfish, crawdad, crawdaddy`, `cowboy hat, ten-gallon hat`, `conch`, `spaghetti squash`, `toy poodle`, `saltshaker, salt shaker`, `microwave, microwave oven`, `triceratops`, `necklace`, `castle`, `streetcar, tram, tramcar, trolley, trolley car`, `eel`, `diaper, nappy, napkin`, `standard poodle`, `prayer rug, prayer mat`, `radio, wireless`, `crane`, `envelope`, `rule, ruler`, `gar, garfish, garpike, billfish, Lepisosteus osseus`, `spider monkey, Ateles geoffroyi`, `Irish wolfhound`, `German shepherd, German shepherd dog, German police dog, alsatian`, `umbrella`, `sunglass`, `aircraft carrier, carrier, flattop, attack aircraft carrier`, `water buffalo, water ox, Asiatic buffalo, Bubalus bubalis`, `jellyfish`, `groom, bridegroom`, `tree frog, tree-frog`, `steel arch bridge`, `lemon`, `pickup, pickup truck`, `vault`, `groenendael`, `baseball`, `junco, snowbird`, `maillot, tank suit`, `gazelle`, `jack-o'-lantern`, `military uniform`, `slide rule, slipstick`, `wire-haired fox terrier`, `acorn squash`, `electric fan, blower`, `Brittany spaniel`, `chimpanzee, chimp, Pan troglodytes`, `pillow`, `binder, ring-binder`, `schipperke`, `Afghan hound, Afghan`, `plate rack`, `car mirror`, `hand-held computer, hand-held microcomputer`, `papillon`, `schooner`, `Bedlington terrier`, `cellular telephone, cellular phone, cellphone, cell, mobile phone`, `altar`, `Chesapeake Bay retriever`, `cabbage butterfly`, `polecat, fitch, foulmart, foumart, Mustela putorius`, `comic book`, `French horn, horn`, `daisy`, `organ, pipe organ`, `mashed potato`, `acorn`, `fly`, `chain`, `American alligator, Alligator mississipiensis`, `mink`, `garbage truck, dustcart`, `totem pole`, `wine bottle`, `strawberry`, `cricket`, `European fire salamander, Salamandra salamandra`, `coral reef`, `Welsh springer spaniel`, `bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis`, `snorkel`, `bald eagle, American eagle, Haliaeetus leucocephalus`, `meerkat, mierkat`, `grille, radiator grille`, `nematode, nematode worm, roundworm`, `anemone fish`, `corn`, `loggerhead, loggerhead turtle, Caretta caretta`, `palace`, `suit, suit of clothes`, `pineapple, ananas`, `macaque`, `ping-pong ball`, `ram, tup`, `church, church building`, `koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus`, `hare`, `bath towel`, `strainer`, `yawl`, `otterhound, otter hound`, `table lamp`, `king snake, kingsnake`, `lotion`, `lion, king of beasts, Panthera leo`, `thatch, thatched roof`, `basset, basset hound`, `black and gold garden spider, Argiope aurantia`, `barber chair`, `proboscis monkey, Nasalis larvatus`, `consomme`, `Irish terrier`, `Irish water spaniel`, `common iguana, iguana, Iguana iguana`, `Weimaraner`, `Great Dane`, `pug, pug-dog`, `rhinoceros beetle`, `vase`, `brass, memorial tablet, plaque`, `kit fox, Vulpes macrotis`, `king penguin, Aptenodytes patagonica`, `vending machine`, `dalmatian, coach dog, carriage dog`, `rock beauty, Holocanthus tricolor`, `pole`, `cuirass`, `bolete`, `jackfruit, jak, jack`, `monarch, monarch butterfly, milkweed butterfly, Danaus plexippus`, `chow, chow chow`, `nail`, `packet`, `half track`, `Lhasa, Lhasa apso`, `boathouse`, `hay`, `valley, vale`, `slot, one-armed bandit`, `volleyball`, `carton`, `shower cap`, `tile roof`, `lacewing, lacewing fly`, `patas, hussar monkey, Erythrocebus patas`, `boa constrictor, Constrictor constrictor`, `black swan, Cygnus atratus`, `lampshade, lamp shade`, `mousetrap`, `crutch`, `vestment`, `Pekinese, Pekingese, Peke`, `tusker`, `warplane, military plane`, `sandal`, `screw`, `custard apple`, `Scottish deerhound, deerhound`, `spindle`, `keeshond`, `hummingbird`, `letter opener, paper knife, paperknife`, `cucumber, cuke`, `bearskin, busby, shako`, `fire engine, fire truck`, `trombone`, `ringneck snake, ring-necked snake, ring snake`, `sorrel`, `fire screen, fireguard`, `paper towel`, `flute, transverse flute`, `hotdog, hot dog, red hot`, `Indian elephant, Elephas maximus`, `mosque`, `stingray`, `Rhodesian ridgeback`, `four-poster` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_small_patch16_224_en_4.1.0_3.0_1660165708246.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_deit_small_patch16_224_en_4.1.0_3.0_1660165708246.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_deit_small_patch16_224", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_deit_small_patch16_224", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_deit_small_patch16_224| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|83.2 MB| --- layout: model title: Detect concepts in drug development trials (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_drug_development_trials date: 2022-06-18 tags: [ner, en, bertfortokenclassification, clinical, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description It is a BertForTokenClassification NER model to identify concepts related to drug development including `Trial Groups` , Efficacy and Safety `End Points` , `Hazard Ratio`, and others in free text. ## Predicted Entities `Hazard_Ratio`, `Confidence_Interval`, `Patient_Count`, `Trial_Group`, `Patient_Group`, `Duration`, `Confidence_level`, `P_Value`, `Confidence_Range`, `End_Point`, `Follow_Up`, `ADE`, `Value`, `DATE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DRUGS_DEVELOPMENT_TRIALS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_drug_development_trials_en_3.4.1_3.0_1655578771078.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_drug_development_trials_en_3.4.1_3.0_1655578771078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_drug_development_trials", "en", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) test_sentence = """In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.""" data = spark.createDataFrame([[test_sentence]]).toDF('text') result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_drug_development_trials", "en", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter)) val data = Seq("In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.drug_development_trials").predict("""In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.""") ```

## Results ```bash +-----------------+-------------+ |chunk |ner_label | +-----------------+-------------+ |median |Duration | |overall survival |End_Point | |with |Trial_Group | |without topotecan|Trial_Group | |4.0 |Value | |3.6 months |Value | |23 |Patient_Count| |63 |Patient_Count| |55 |Patient_Count| |33 patients |Patient_Count| |topotecan |Trial_Group | |11 |Patient_Count| |61 |Patient_Count| |66 |Patient_Count| |32 patients |Patient_Count| |without topotecan|Trial_Group | +-----------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_drug_development_trials| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|400.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References Trained on data obtained from `clinicaltrials.gov` and annotated in-house. ## Benchmarking ```bash label prec rec f1 support B-ADE 0.50 0.33 0.40 3 B-Confidence_Interval 0.46 1.00 0.63 12 B-Confidence_Range 1.00 0.98 0.99 42 B-Confidence_level 1.00 0.67 0.81 43 B-DATE 0.95 0.93 0.94 40 B-Duration 1.00 0.82 0.90 11 B-End_Point 0.91 0.98 0.95 54 B-Follow_Up 1.00 1.00 1.00 2 B-Hazard_Ratio 0.77 1.00 0.87 24 B-P_Value 1.00 0.56 0.71 9 B-Patient_Count 1.00 0.95 0.97 19 B-Patient_Group 0.79 0.63 0.70 43 B-Trial_Group 0.96 0.94 0.95 274 B-Value 0.98 0.83 0.90 77 I-ADE 0.71 1.00 0.83 12 I-Confidence_Range 0.98 1.00 0.99 43 I-DATE 0.95 1.00 0.98 60 I-Duration 1.00 1.00 1.00 1 I-End_Point 0.92 1.00 0.96 44 I-Follow_Up 1.00 1.00 1.00 2 I-P_Value 0.82 1.00 0.90 18 I-Patient_Count 0.00 0.00 0.00 0 I-Patient_Group 0.79 0.94 0.86 187 I-Trial_Group 0.92 0.90 0.91 156 I-Value 1.00 1.00 1.00 10 O 0.98 0.98 0.98 2622 accuracy - - 0.96 3808 macro-avg 0.86 0.86 0.85 3808 weighted-avg 0.96 0.96 0.96 3808 ``` --- layout: model title: Legal Insolvency Clause Binary Classifier author: John Snow Labs name: legclf_insolvency_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `insolvency` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `insolvency` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_insolvency_clause_en_1.0.0_3.2_1660122549928.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_insolvency_clause_en_1.0.0_3.2_1660122549928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_insolvency_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[insolvency]| |[other]| |[other]| |[insolvency]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_insolvency_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support insolvency 0.98 0.98 0.98 43 other 0.99 0.99 0.99 101 accuracy - - 0.99 144 macro-avg 0.98 0.98 0.98 144 weighted-avg 0.99 0.99 0.99 144 ``` --- layout: model title: Dutch BertForQuestionAnswering model (from henryk) author: John Snow Labs name: bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2 date: 2022-06-02 tags: [nl, open_source, question_answering, bert] task: Question Answering language: nl edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-dutch-squad2` is a Dutch model orginally trained by `henryk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2_nl_4.0.0_3.0_1654180038184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2_nl_4.0.0_3.0_1654180038184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2","nl") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2","nl") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("nl.answer_question.squadv2.bert.multilingual_base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_cased_finetuned_dutch_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|nl| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/henryk/bert-base-multilingual-cased-finetuned-dutch-squad2 - https://www.linkedin.com/in/henryk-borzymowski-0755a2167/ - https://rajpurkar.github.io/SQuAD-explorer/ - https://github.com/google-research/bert/blob/master/multilingual.md --- layout: model title: Ukrainian XLMRoBerta Embeddings (from ukr-models) author: John Snow Labs name: xlmroberta_embeddings_xlm_roberta_base date: 2022-05-14 tags: [open_source, embeddings, uk, xlm_roberta] task: Embeddings language: uk edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-uk` is a Ukrainian model orginally trained by `ukr-models`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_xlm_roberta_base_uk_3.4.4_3.0_1652533212195.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_xlm_roberta_base_uk_3.4.4_3.0_1652533212195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_xlm_roberta_base| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|uk| |Size:|264.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/ukr-models/xlm-roberta-base-uk --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from 21iridescent) author: John Snow Labs name: roberta_qa_base_finetuned_squad2_lwt date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-base-finetuned-squad2-lwt` is a English model originally trained by `21iridescent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_squad2_lwt_en_4.3.0_3.0_1674210398377.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_squad2_lwt_en_4.3.0_3.0_1674210398377.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_squad2_lwt","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_squad2_lwt","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_squad2_lwt| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|307.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/21iridescent/distilroberta-base-finetuned-squad2-lwt --- layout: model title: Chinese BertForQuestionAnswering model (from voidful) author: John Snow Labs name: bert_qa_question_answering_zh_voidful date: 2022-06-03 tags: [open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `question-answering-zh` is a Chinese model orginally trained by `voidful`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_zh_voidful_zh_4.0.0_3.0_1654250011767.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_zh_voidful_zh_4.0.0_3.0_1654250011767.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_question_answering_zh_voidful","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_question_answering_zh_voidful","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.by_voidful").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_question_answering_zh_voidful| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|381.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/voidful/question-answering-zh --- layout: model title: Russian T5ForConditionalGeneration Cased model (from IlyaGusev) author: John Snow Labs name: t5_sber_rut5_filler date: 2023-01-30 tags: [ru, open_source, t5, tensorflow] task: Text Generation language: ru edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sber_rut5_filler` is a Russian model originally trained by `IlyaGusev`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_sber_rut5_filler_ru_4.3.0_3.0_1675107176054.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_sber_rut5_filler_ru_4.3.0_3.0_1675107176054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_sber_rut5_filler","ru") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_sber_rut5_filler","ru") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_sber_rut5_filler| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ru| |Size:|927.4 MB| ## References - https://huggingface.co/IlyaGusev/sber_rut5_filler --- layout: model title: Bengali Lemmatizer author: John Snow Labs name: lemma date: 2021-01-20 task: Lemmatization language: bn edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [bn, lemmatizer, open_source] supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_bn_2.7.0_2.4_1611163691269.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_bn_2.7.0_2.4_1611163691269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols(["document"]) \ .setOutputCol("token") lemmatizer = LemmatizerModel.pretrained("lemma", "bn") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) results = light_pipeline.fullAnnotate(["একদিন প্রাতে বৈদ্যনাথের মার্বলমণ্ডিত দালানে একটি স্থূলোদর সন্ন্যাসী দুইসের মোহনভোগ এবং দেড়সের দুগ্ধ সেবায় নিযুক্ত আছে বৈদ্যনাথ গায়ে একখানি চাদর দিয়া জোড়করে একান্ত বিনীতভাবে ভূতলে বসিয়া ভক্তিভরে পবিত্র ভোজনব্যাপার নিরীক্ষণ করিতেছিলেন এমন সময় কোনোমতে দ্বারীদের দৃষ্টি এড়াইয়া জীর্ণদেহ বালক সহিত একটি অতি শীর্ণকায়া রমণী গৃহে প্রবেশ করিয়া ক্ষীণস্বরে কহিল বাবু দুটি খেতে দাও"]) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = Tokenizer() .setInputCols("document") .setOutputCol("token") val lemmatizer = LemmatizerModel.pretrained("lemma", "bn") .setInputCols("token") .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("একদিন প্রাতে বৈদ্যনাথের মার্বলমণ্ডিত দালানে একটি স্থূলোদর সন্ন্যাসী দুইসের মোহনভোগ এবং দেড়সের দুগ্ধ সেবায় নিযুক্ত আছে বৈদ্যনাথ গায়ে একখানি চাদর দিয়া জোড়করে একান্ত বিনীতভাবে ভূতলে বসিয়া ভক্তিভরে পবিত্র ভোজনব্যাপার নিরীক্ষণ করিতেছিলেন এমন সময় কোনোমতে দ্বারীদের দৃষ্টি এড়াইয়া জীর্ণদেহ বালক সহিত একটি অতি শীর্ণকায়া রমণী গৃহে প্রবেশ করিয়া ক্ষীণস্বরে কহিল বাবু দুটি খেতে দাও").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["একদিন প্রাতে বৈদ্যনাথের মার্বলমণ্ডিত দালানে একটি স্থূলোদর সন্ন্যাসী দুইসের মোহনভোগ এবং দেড়সের দুগ্ধ সেবায় নিযুক্ত আছে বৈদ্যনাথ গায়ে একখানি চাদর দিয়া জোড়করে একান্ত বিনীতভাবে ভূতলে বসিয়া ভক্তিভরে পবিত্র ভোজনব্যাপার নিরীক্ষণ করিতেছিলেন এমন সময় কোনোমতে দ্বারীদের দৃষ্টি এড়াইয়া জীর্ণদেহ বালক সহিত একটি অতি শীর্ণকায়া রমণী গৃহে প্রবেশ করিয়া ক্ষীণস্বরে কহিল বাবু দুটি খেতে দাও"] lemma_df = nlu.load('bn.lemma').predict(text, output_level = "document") lemma_df.lemma.values[0] ```

## Results ```bash {'lemma': [Annotation(token, 0, 4, একদিন, {'sentence': '0'}), Annotation(token, 6, 11, প্রাতঃ, {'sentence': '0'}), Annotation(token, 13, 22, বৈদ্যনাথ, {'sentence': '0'}), Annotation(token, 24, 35, মার্বলমণ্ডিত, {'sentence': '0'}), Annotation(token, 37, 42, দালান, {'sentence': '0'}), Annotation(token, 44, 47, এক, {'sentence': '0'}), Annotation(token, 49, 56, স্থূলউদর, {'sentence': '0'}), Annotation(token, 58, 66, সন্ন্যাসী, {'sentence': '0'}), Annotation(token, 68, 73, দুইসের, {'sentence': '0'}), Annotation(token, 75, 81, মোহনভোগ, {'sentence': '0'}), Annotation(token, 83, 85, এবং, {'sentence': '0'}), Annotation(token, 87, 93, দেড়সের, {'sentence': '0'}), Annotation(token, 95, 99, দুগ্ধ, {'sentence': '0'}), Annotation(token, 101, 105, সেবা, {'sentence': '0'}), Annotation(token, 107, 113, নিযুক্ত, {'sentence': '0'}), Annotation(token, 115, 117, আছে, {'sentence': '0'}), Annotation(token, 119, 126, বৈদ্যনাথ, {'sentence': '0'}), Annotation(token, 128, 131, গা, {'sentence': '0'}), Annotation(token, 133, 138, একখান, {'sentence': '0'}), Annotation(token, 140, 143, চাদর, {'sentence': '0'}), Annotation(token, 145, 148, দেওয়া, {'sentence': '0'}), Annotation(token, 150, 156, জোড়কর, {'sentence': '0'}), Annotation(token, 158, 163, একান্ত, {'sentence': '0'}), Annotation(token, 165, 173, বিনীতভাব, {'sentence': '0'}), Annotation(token, 175, 179, ভূতল, {'sentence': '0'}), Annotation(token, 181, 185, বসা, {'sentence': '0'}), Annotation(token, 187, 194, ভক্তিভরা, {'sentence': '0'}), Annotation(token, 196, 201, পবিত্র, {'sentence': '0'}), Annotation(token, 203, 213, ভোজনব্যাপার, {'sentence': '0'}), Annotation(token, 215, 222, নিরীক্ষণ, {'sentence': '0'}), Annotation(token, 224, 233, করা, {'sentence': '0'}), Annotation(token, 235, 237, এমন, {'sentence': '0'}), Annotation(token, 239, 241, সময়, {'sentence': '0'}), Annotation(token, 243, 249, কোনোমত, {'sentence': '0'}), Annotation(token, 251, 259, দ্বারী, {'sentence': '0'}), Annotation(token, 261, 266, দৃষ্টি, {'sentence': '0'}), Annotation(token, 268, 274, এড়ানো, {'sentence': '0'}), Annotation(token, 276, 283, জীর্ণদেহ, {'sentence': '0'}), Annotation(token, 285, 288, বালক, {'sentence': '0'}), Annotation(token, 290, 293, সহিত, {'sentence': '0'}), Annotation(token, 295, 298, এক, {'sentence': '0'}), Annotation(token, 300, 302, অতি, {'sentence': '0'}), Annotation(token, 304, 312, শীর্ণকায়া, {'sentence': '0'}), Annotation(token, 314, 317, রমণী, {'sentence': '0'}), Annotation(token, 319, 322, গৃহ, {'sentence': '0'}), Annotation(token, 324, 329, প্রবেশ, {'sentence': '0'}), Annotation(token, 331, 335, বিশ্বাস, {'sentence': '0'}), Annotation(token, 337, 346, ক্ষীণস্বর, {'sentence': '0'}), Annotation(token, 348, 351, কহা, {'sentence': '0'}), Annotation(token, 353, 356, বাবু, {'sentence': '0'}), Annotation(token, 358, 361, দুই, {'sentence': '0'}), Annotation(token, 363, 366, খাওয়া, {'sentence': '0'}), Annotation(token, 368, 370, দাওয়া, {'sentence': '0'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|bn| ## Data Source The model was trained on the annotated Bengali data set from the [Indian Statistics Institute](https://www.isical.ac.in). Reference: - A. Chakrabarty, O.A. Pandit, and U. Garain (2017): Context Sensitive Lemmatization Using Two Successive Bidirectional Gated Recurrent Networks, in ACL 2017:1481-1491. --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_v2 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-robertav2` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_v2_es_4.3.0_3.0_1674218559260.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_v2_es_4.3.0_3.0_1674218559260.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_v2","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_v2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_v2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-robertav2 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from adi1494) author: John Snow Labs name: distilbert_qa_adi1494_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `adi1494`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_adi1494_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769689530.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_adi1494_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769689530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_adi1494_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_adi1494_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_adi1494_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/adi1494/distilbert-base-uncased-finetuned-squad --- layout: model title: Twi T5ForConditionalGeneration Cased model (from clhuang) author: John Snow Labs name: t5_hotel_review_sentiment date: 2023-01-31 tags: [tw, open_source, t5, tensorflow] task: Text Generation language: tw edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-hotel-review-sentiment` is a Twi model originally trained by `clhuang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_hotel_review_sentiment_tw_4.3.0_3.0_1675124731516.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_hotel_review_sentiment_tw_4.3.0_3.0_1675124731516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_hotel_review_sentiment","tw") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_hotel_review_sentiment","tw") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_hotel_review_sentiment| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|tw| |Size:|1.0 GB| ## References - https://huggingface.co/clhuang/t5-hotel-review-sentiment --- layout: model title: Word Embeddings for Persian (persian_w2v_cc_300d) author: John Snow Labs name: persian_w2v_cc_300d date: 2020-12-05 task: Embeddings language: fa edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [embeddings, fa, open_source] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/persian_w2v_cc_300d_fa_2.7.0_2.4_1607169840793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/persian_w2v_cc_300d_fa_2.7.0_2.4_1607169840793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("persian_w2v_cc_300d", "fa") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['من یادگیری ماشین را دوست دارم']], ["text"])) ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("persian_w2v_cc_300d", "fa") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("من یادگیری ماشین را دوست دارم").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""من یادگیری ماشین را دوست دارم"""] farvec_df = nlu.load('fa.embed.word2vec.300d').predict(text, output_level='token') farvec_df ```

{:.h2_title} ## Results The model gives 300 dimensional Word2Vec feature vector outputs per token. ```bash | token | fa_embed_word2vec_300d_embeddings |-------|-------------------------------------------------- | من | [-0.3861289620399475, -0.08295578509569168, -0... | را | [-0.15430298447608948, -0.24924889206886292, 0... | دوست | [0.07587642222642899, -0.24341894686222076, 0.... | دارم | [0.0899219810962677, -0.21863090991973877, 0.4... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|persian_w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|fa| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: Legal Introduction Clause Binary Classifier author: John Snow Labs name: legclf_introduction_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `introduction` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `introduction` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_en_1.0.0_3.2_1660122579442.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_introduction_clause_en_1.0.0_3.2_1660122579442.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_introduction_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lorenzkuhn_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lorenzkuhn_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_lorenzkuhn_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/lorenzkuhn/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate Kwangali to English Pipeline author: John Snow Labs name: translate_kwn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, kwn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `kwn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_kwn_en_xx_2.7.0_2.4_1609698648068.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_kwn_en_xx_2.7.0_2.4_1609698648068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_kwn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_kwn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.kwn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_kwn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish RoBERTa Embeddings (Base, Random Sampling) author: John Snow Labs name: roberta_embeddings_bertin_base_random date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bertin-base-random` is a Spanish model orginally trained by `bertin-project`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_random_es_3.4.2_3.0_1649945870509.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_random_es_3.4.2_3.0_1649945870509.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bertin_base_random| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|234.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/bertin-project/bertin-base-random --- layout: model title: English asr_wav2vec2_large_xlsr_ksponspeech_1_20 TFWav2Vec2ForCTC from cheulyop author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_ksponspeech_1_20` is a English model originally trained by cheulyop. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20_en_4.2.0_3.0_1664097449426.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20_en_4.2.0_3.0_1664097449426.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_ksponspeech_1_20| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from internetoftim) author: John Snow Labs name: bert_qa_demo date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `demo` is a English model orginally trained by `internetoftim`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_demo_en_4.0.0_3.0_1654187421235.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_demo_en_4.0.0_3.0_1654187421235.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_demo","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_demo","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_internetoftim").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_demo| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|798.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/internetoftim/demo --- layout: model title: Translate Indo-European languages to English Pipeline author: John Snow Labs name: translate_ine_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ine, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ine` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ine_en_xx_2.7.0_2.4_1609691766618.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ine_en_xx_2.7.0_2.4_1609691766618.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ine_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ine_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ine.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ine_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Stop Words Cleaner for Anglo-French author: John Snow Labs name: stopwords_af date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: af edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, af] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_af_af_2.5.4_2.4_1594742440083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_af_af_2.5.4_2.4_1594742440083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_af", "af") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Anders as die koning van die noorde, is John Snow 'n Engelse dokter en 'n leier in die ontwikkeling van narkose en mediese higiëne.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_af", "af") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Anders as die koning van die noorde, is John Snow "n Engelse dokter en "n leier in die ontwikkeling van narkose en mediese higiëne.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Anders as die koning van die noorde, is John Snow 'n Engelse dokter en 'n leier in die ontwikkeling van narkose en mediese higiëne."""] stopword_df = nlu.load('af.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=5, result='Anders', metadata={'sentence': '0'}), Row(annotatorType='token', begin=14, end=19, result='koning', metadata={'sentence': '0'}), Row(annotatorType='token', begin=29, end=34, result='noorde', metadata={'sentence': '0'}), Row(annotatorType='token', begin=35, end=35, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=40, end=43, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_af| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|af| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from doc2query) author: John Snow Labs name: t5_stackexchange_title_body_small_v1 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `stackexchange-title-body-t5-small-v1` is a English model originally trained by `doc2query`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_stackexchange_title_body_small_v1_en_4.3.0_3.0_1675107510591.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_stackexchange_title_body_small_v1_en_4.3.0_3.0_1675107510591.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_stackexchange_title_body_small_v1","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_stackexchange_title_body_small_v1","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_stackexchange_title_body_small_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|347.9 MB| ## References - https://huggingface.co/doc2query/stackexchange-title-body-t5-small-v1 - https://arxiv.org/abs/1904.08375 - https://cs.uwaterloo.ca/~jimmylin/publications/Nogueira_Lin_2019_docTTTTTquery-v2.pdf - https://arxiv.org/abs/2104.08663 - https://github.com/UKPLab/beir - https://www.sbert.net/examples/unsupervised_learning/query_generation/README.html --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes author: John Snow Labs name: sbiobertresolve_umls_findings date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to 4 major categories of UMLS CUI codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities This model returns CUI (concept unique identifier) codes for 200K concepts from clinical findings.https://www.nlm.nih.gov/research/umls/index.html {:.btn-box} [Live Demo](http://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_3.0.4_3.0_1621189546348.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_3.0.4_3.0_1621189546348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_cased_IUChatbot_ontologyDts","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_cased_IUChatbot_ontologyDts","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.base_cased.by_nntadotzip").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_cased_IUChatbot_ontologyDts| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nntadotzip/bert-base-cased-IUChatbot-ontologyDts --- layout: model title: SDOH Insurance Status For Classification author: John Snow Labs name: genericclassifier_sdoh_insurance_status_sbiobert_cased_mli date: 2023-04-27 tags: [en, licensed, sdoh, social_determinants, generic_classifier, biobert] task: Text Classification language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting whether the patient has insurance or not. If the patient's insurance status is not mentioned or is unknown, it is regarded as "Unknown". The model is trained by using GenericClassifierApproach annotator. `Insured`: The patient has insurance. `Uninsured`: The patient has no insurance. `Unknown`: Insurance status is not mentioned in the clinical notes or is unknown. ## Predicted Entities `Insured`, `Uninsured`, `Unknown` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/social_determinant){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_insurance_status_sbiobert_cased_mli_en_4.4.0_3.0_1682623268182.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_insurance_status_sbiobert_cased_mli_en_4.4.0_3.0_1682623268182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+-----------+ | text| result| +----------------------------------------------------------------------------------------------------+-----------+ | The patient has VA insurance.| [Insured]| | She doesn't have any kind of insurance|[Uninsured]| |Patient: Mary H.\n\nBackground: Mary is a 40-year-old woman who has been diagnosed with asthma an...| [Insured]| |Patient: Sarah L.\n\nBackground: Sarah is a 35-year-old woman who has been experiencing housing i...|[Uninsured]| +----------------------------------------------------------------------------------------------------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_insurance_status_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References SDOH internal project ## Benchmarking ```bash label precision recall f1-score support Insured 0.94 0.90 0.92 145 Uninsured 0.84 0.89 0.86 36 Unknown 0.72 0.82 0.77 38 accuracy - - 0.88 219 macro-avg 0.84 0.87 0.85 219 weighted-avg 0.89 0.88 0.88 219 ``` --- layout: model title: English image_classifier_vit_age_classifier ViTForImageClassification from ibombonato author: John Snow Labs name: image_classifier_vit_age_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_age_classifier` is a English model originally trained by ibombonato. ## Predicted Entities `60-70`, `40-50`, `70-80`, `80-90`, `10-20`, `50-60`, `90-100`, `30-40`, `0-10`, `20-30` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_age_classifier_en_4.1.0_3.0_1660169996525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_age_classifier_en_4.1.0_3.0_1660169996525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_age_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_age_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_age_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English T5ForConditionalGeneration Cased model (from gagan3012) author: John Snow Labs name: t5_gagan3012_k2t_test date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `k2t-test` is a English model originally trained by `gagan3012`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_gagan3012_k2t_test_en_4.3.0_3.0_1675103953989.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_gagan3012_k2t_test_en_4.3.0_3.0_1675103953989.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_gagan3012_k2t_test","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_gagan3012_k2t_test","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_gagan3012_k2t_test| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|242.7 MB| ## References - https://huggingface.co/gagan3012/k2t-test - https://pypi.org/project/keytotext/ - https://pepy.tech/project/keytotext - https://colab.research.google.com/github/gagan3012/keytotext/blob/master/notebooks/K2T.ipynb - https://share.streamlit.io/gagan3012/keytotext/UI/app.py - https://github.com/gagan3012/keytotext#api - https://hub.docker.com/r/gagan30/keytotext - https://keytotext.readthedocs.io/en/latest/?badge=latest - https://github.com/psf/black - https://socialify.git.ci/gagan3012/keytotext/image?description=1&forks=1&language=1&owner=1&stargazers=1&theme=Light --- layout: model title: Translate Semitic languages to English Pipeline author: John Snow Labs name: translate_sem_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, sem, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `sem` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_sem_en_xx_2.7.0_2.4_1609686594071.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_sem_en_xx_2.7.0_2.4_1609686594071.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_sem_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_sem_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.sem.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_sem_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Korean Electra Embeddings (from monologg) author: John Snow Labs name: electra_embeddings_koelectra_base_generator date: 2022-05-17 tags: [ko, open_source, electra, embeddings] task: Embeddings language: ko edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `koelectra-base-generator` is a Korean model orginally trained by `monologg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_koelectra_base_generator_ko_3.4.4_3.0_1652786875395.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_koelectra_base_generator_ko_3.4.4_3.0_1652786875395.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_koelectra_base_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ko| |Size:|130.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/monologg/koelectra-base-generator - https://github.com/monologg/KoELECTRA/blob/master/README_EN.md --- layout: model title: Legal Capital Expenditures Clause Binary Classifier author: John Snow Labs name: legclf_capital_expenditures_clause date: 2023-01-27 tags: [en, legal, classification, capital, expenditures, clauses, capital_expenditures, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `capital-expenditures` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `capital-expenditures`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_capital_expenditures_clause_en_1.0.0_3.0_1674820293638.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_capital_expenditures_clause_en_1.0.0_3.0_1674820293638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_capital_expenditures_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[capital-expenditures]| |[other]| |[other]| |[capital-expenditures]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_capital_expenditures_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support capital-expenditures 1.00 0.97 0.99 37 other 0.97 1.00 0.99 38 accuracy - - 0.99 75 macro-avg 0.99 0.99 0.99 75 weighted-avg 0.99 0.99 0.99 75 ``` --- layout: model title: English asr_wav2vec2_base_10k_voxpopuli TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_base_10k_voxpopuli date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_10k_voxpopuli` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_10k_voxpopuli_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_10k_voxpopuli_en_4.2.0_3.0_1664022541668.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_10k_voxpopuli_en_4.2.0_3.0_1664022541668.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_10k_voxpopuli', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_10k_voxpopuli", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_10k_voxpopuli| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: French CamemBert Embeddings (from mdroth) author: John Snow Labs name: camembert_embeddings_generic_model_R91m date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model_R91m` is a French model orginally trained by `mdroth`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_generic_model_R91m_fr_3.4.4_3.0_1653991354882.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_generic_model_R91m_fr_3.4.4_3.0_1653991354882.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_generic_model_R91m| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/mdroth/dummy-model_R91m --- layout: model title: Lemmatizer (Swedish, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, sv] task: Lemmatization language: sv edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Swedish Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_sv_3.4.1_3.0_1646316517673.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_sv_3.4.1_3.0_1646316517673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+ |result | +------------------------------+ |[Du, vara, inte, god, än, jag]| +------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|sv| |Size:|8.0 MB| --- layout: model title: English asr_wav2vec2_large_xls_r_300m_georgian_large TFWav2Vec2ForCTC from RaphaelKalandadze author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_georgian_large date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_georgian_large` is a English model originally trained by RaphaelKalandadze. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_georgian_large_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_georgian_large_en_4.2.0_3.0_1664120201846.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_georgian_large_en_4.2.0_3.0_1664120201846.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_georgian_large", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_georgian_large", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_georgian_large| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Spanish RobertaForQuestionAnswering (from IIC) author: John Snow Labs name: roberta_qa_roberta_base_spanish_squades date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades` is a Spanish model originally trained by `IIC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_spanish_squades_es_4.0.0_3.0_1655734721755.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_spanish_squades_es_4.0.0_3.0_1655734721755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_spanish_squades","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_spanish_squades","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squad.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_spanish_squades| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|459.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/IIC/roberta-base-spanish-squades - https://paperswithcode.com/sota?task=question-answering&dataset=squad_es - https://arxiv.org/abs/2107.07253 --- layout: model title: Castilian, Spanish BertForQuestionAnswering model (from CenIA) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-qa-mlqa` is a Castilian, Spanish model orginally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa_es_4.0.0_3.0_1654180397621.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa_es_4.0.0_3.0_1654180397621.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.mlqa.bert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_qa_mlqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/bert-base-spanish-wwm-cased-finetuned-qa-mlqa --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_squad_3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad-3` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_squad_3_en_4.0.0_3.0_1655734505387.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_squad_3_en_4.0.0_3.0_1655734505387.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_squad_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_squad_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_v3.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_squad_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-squad-3 --- layout: model title: English image_classifier_vit_mit_indoor_scenes ViTForImageClassification from vincentclaes author: John Snow Labs name: image_classifier_vit_mit_indoor_scenes date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_mit_indoor_scenes` is a English model originally trained by vincentclaes. ## Predicted Entities `airport_inside`, `bowling`, `buffet`, `movietheater`, `clothingstore`, `inside_bus`, `fastfood_restaurant`, `operating_room`, `corridor`, `cloister`, `stairscase`, `auditorium`, `meeting_room`, `livingroom`, `videostore`, `bathroom`, `inside_subway`, `bedroom`, `casino`, `tv_studio`, `classroom`, `laboratorywet`, `nursery`, `office`, `deli`, `prisoncell`, `dentaloffice`, `restaurant_kitchen`, `studiomusic`, `locker_room`, `restaurant`, `laundromat`, `dining_room`, `subway`, `gameroom`, `museum`, `mall`, `garage`, `elevator`, `jewelleryshop`, `kindergarden`, `toystore`, `concert_hall`, `artstudio`, `kitchen`, `florist`, `waitingroom`, `grocerystore`, `library`, `bar`, `computerroom`, `trainstation`, `lobby`, `church_inside`, `pantry`, `closet`, `children_room`, `hairsalon`, `shoeshop`, `greenhouse`, `bookstore`, `bakery`, `poolinside`, `warehouse`, `winecellar`, `hospitalroom`, `gym` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_mit_indoor_scenes_en_4.1.0_3.0_1660170571863.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_mit_indoor_scenes_en_4.1.0_3.0_1660170571863.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_mit_indoor_scenes", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_mit_indoor_scenes", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_mit_indoor_scenes| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.1 MB| --- layout: model title: Generic Deidentification NER author: John Snow Labs name: finner_deid date: 2022-08-09 tags: [en, finance, ner, deid, licensed] task: [De-identification, Named Entity Recognition] language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model which allows you to detect some generic entities that may require to be masked or obfuscated to be compliant with different regulations, as GDPR and CCPA. This is just an NER model, make sure you try the full De-identification pipelines available in Models Hub. ## Predicted Entities `AGE`, `CITY`, `COUNTRY`, `DATE`, `EMAIL`, `FAX`, `LOCATION-OTHER`, `ORG`, `PERSON`, `PHONE`, `PROFESSION`, `STATE`, `STREET`, `URL`, `ZIP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/DEID_FIN/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_deid_en_1.0.0_3.2_1660050720560.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_deid_en_1.0.0_3.2_1660050720560.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+----------------+ | token| ner_label| +-----------+----------------+ | This| O| | LICENSE| O| | AND| O| |DEVELOPMENT| O| | AGREEMENT| O| | (| O| | this| O| | Agreement| O| | )| O| | is| O| | entered| O| | into| O| | effective| O| | as| O| | of| O| | Nov| B-DATE| | .| I-DATE| | 02| I-DATE| | ,| I-DATE| | 2019| I-DATE| | (| O| | the| O| | Effective| O| | Date| O| | )| O| | by| O| | and| O| | between| O| | Bioeq| O| | IP| O| | AG| O| | ,| O| | having| O| | its| O| | principal| O| | place| O| | of| O| | business| O| | at| O| | 333| B-STREET| | Twin| I-STREET| | Dolphin| I-STREET| | Drive| I-STREET| | ,| O| | Suite|B-LOCATION-OTHER| | 600|I-LOCATION-OTHER| | ,| O| | Redwood| B-CITY| | City| I-CITY| | ,| O| | CA| B-STATE| | ,| O| | 94065| B-ZIP| | ,| O| | USA| B-STATE| | (| O| | Licensee| O| | ).| O| +-----------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_deid| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.4 MB| ## References In-house annotated documents with protected information ## Benchmarking ```bash label precision recall f1-score support B-AGE 0.96 0.89 0.92 245 B-CITY 0.85 0.86 0.86 123 B-COUNTRY 0.86 0.67 0.75 36 B-DATE 0.98 0.97 0.97 2352 B-ORG 0.75 0.71 0.73 38 B-PERSON 0.97 0.94 0.95 1348 B-PHONE 0.86 0.80 0.83 86 B-PROFESSION 0.93 0.75 0.83 84 B-STATE 0.92 0.89 0.91 102 B-STREET 0.99 0.91 0.95 89 I-CITY 0.82 0.77 0.79 35 I-COUNTRY 1.00 0.50 0.67 6 I-DATE 0.96 0.95 0.96 402 I-ORG 0.71 0.86 0.77 28 I-PERSON 0.98 0.96 0.97 1240 I-PHONE 0.91 0.92 0.92 77 I-PROFESSION 0.96 0.79 0.87 70 I-STATE 1.00 0.62 0.77 8 I-STREET 0.98 0.94 0.96 188 I-ZIP 0.84 0.97 0.90 60 O 1.00 1.00 1.00 194103 accuracy - - 1.00 200762 macro-avg 0.72 0.62 0.65 200762 weighted-avg 1.00 1.00 1.00 200762 ``` --- layout: model title: Clinical QA BioGPT (JSL) author: John Snow Labs name: biogpt_chat_jsl date: 2023-04-12 tags: [licensed, en, clinical, text_generation, biogpt, tensorflow] task: Text Generation language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalTextGenerator article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is based on BioGPT finetuned with medical conversations happening in a clinical settings and can answer clinical questions related to symptoms, drugs, tests, and diseases. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/BIOGPT_CHAT_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/33.Biogpt_Chat_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/biogpt_chat_jsl_en_4.3.2_3.0_1681319163583.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/biogpt_chat_jsl_en_4.3.2_3.0_1681319163583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['Asthma is itself an allergic disease due to cold or dust or pollen or grass etc. irrespective of the triggering factor. You can go for pulmonary function tests if not done. Treatment is mainly symptomatic which might require inhalation steroids, beta agonists, anticholinergics as MDI or rota haler as a regular treatment. To decrease the inflammation of bronchi and bronchioles, you might be given oral antihistamines with mast cell stabilizers (montelukast) and steroids (prednisolone) with nebulization and frequently steam inhalation. To decrease the bronchoconstriction caused by allergens, you might be given oral antihistamines with mast cell stabilizers (montelukast) and steroids (prednisolone) with nebulization and frequently steam inhalation. The best way to cure any allergy is a complete avoidance of allergen or triggering factor. Consult your pulmonologist for further advise.'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biogpt_chat_jsl| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.4 GB| |Case sensitive:|true| --- layout: model title: English BertForQuestionAnswering Large Uncased model (from michaelrglass) author: John Snow Labs name: bert_qa_large_uncased_ssp date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-sspt` is a English model originally trained by `michaelrglass`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_large_uncased_ssp_en_4.0.0_3.0_1657187762696.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_large_uncased_ssp_en_4.0.0_3.0_1657187762696.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_large_uncased_ssp","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_large_uncased_ssp","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_large_uncased_ssp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|796.4 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/michaelrglass/bert-large-uncased-sspt --- layout: model title: Detect Diagnosis, Symptoms, Drugs, Labs and Demographics (ner_jsl_enriched) author: John Snow Labs name: ner_jsl_enriched date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. Definitions of Predicted Entities: - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_Name`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Frequency`: Frequency of administration for a dose prescribed. - `Gender`: Gender-specific nouns and pronouns. - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Allergen`: Allergen related extractions mentioned in the document. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Respiration`: Number of breaths per minute. - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Temperature`: All mentions that refer to body temperature. - `Weight`: All mentions related to a patients weight. - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). ## Predicted Entities `Age`, `Name`, `Procedure`, `Pulse_Rate`, `Temperature`, `Gender`, `Frequency`, `Route`, `Diagnosis`, `Allergenic_substance`, `Weight`, `Respiratory_Rate`, `Symptom_Name`, `Causative_Agents(Virus_and_Bacteria)`, `Modifier`, `Blood_Pressure`, `Dosage`, `Drug_Name`, `Negation`, `Lab_Result`, `Section_Name`, `Maybe`, `Substance_Name`, `Procedure_Name`, `Lab_Name`, `O2_Saturation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_en_3.0.0_3.0_1617209691808.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_en_3.0.0_3.0_1617209691808.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+------------+ |chunk |ner | +---------------------------+------------+ |21-day-old |Age | |male |Gender | |congestion |Symptom_Name| |mom |Gender | |suctioning yellow discharge|Symptom_Name| |she |Gender | |problems with his breathing|Symptom_Name| |perioral cyanosis |Symptom_Name| |retractions |Symptom_Name| |mom |Gender | |Tylenol |Drug_Name | |His |Gender | |his |Gender | |respiratory congestion |Symptom_Name| |He |Gender | |tired |Symptom_Name| |fussy |Symptom_Name| |albuterol |Drug_Name | +---------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_enriched| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash label tp fp fn prec rec f1 B-Pulse_Rate 80 26 9 0.754717 0.898876 0.820513 I-Diagnosis 2341 1644 1129 0.587453 0.67464 0.628035 I-Procedure_Name 2209 1128 1085 0.661972 0.670613 0.666265 B-Lab_Result 432 107 263 0.801484 0.621583 0.700162 B-Dosage 465 179 81 0.72205 0.851648 0.781513 I-Causative_Agents_(Virus_and_Bacteria) 9 3 10 0.75 0.473684 0.580645 B-Name 648 295 510 0.687169 0.559585 0.616849 I-Name 917 427 665 0.682292 0.579646 0.626794 B-Weight 52 25 9 0.675325 0.852459 0.753623 B-Symptom_Name 4244 1911 1776 0.689521 0.704983 0.697166 I-Maybe 25 15 63 0.625 0.284091 0.390625 I-Symptom_Name 1920 1584 2503 0.547945 0.434095 0.48442 B-Modifier 1399 704 942 0.66524 0.597608 0.629613 B-Blood_Pressure 82 21 7 0.796117 0.921348 0.854167 B-Frequency 290 93 97 0.75718 0.749354 0.753247 I-Gender 29 19 25 0.604167 0.537037 0.568627 I-Age 3 6 11 0.333333 0.214286 0.26087 B-Drug_Name 1762 500 271 0.778957 0.866699 0.820489 B-Substance_Name 143 32 53 0.817143 0.729592 0.770889 B-Temperature 58 23 11 0.716049 0.84058 0.773333 B-Section_Name 2700 294 177 0.901804 0.938478 0.919775 I-Route 131 165 177 0.442568 0.425325 0.433775 B-Maybe 108 47 164 0.696774 0.397059 0.505855 B-Gender 5156 685 68 0.882726 0.986983 0.931948 I-Dosage 435 182 87 0.705024 0.833333 0.763828 B-Causative_Agents_(Virus_and_Bacteria) 21 17 6 0.552632 0.777778 0.646154 I-Frequency 278 131 191 0.679707 0.592751 0.633257 B-Age 352 34 21 0.911917 0.9437 0.927536 I-Lab_Result 27 20 170 0.574468 0.137056 0.221311 B-Negation 1501 311 341 0.828366 0.814875 0.821565 B-Diagnosis 2657 1281 1049 0.674708 0.716945 0.695186 I-Section_Name 3876 1304 188 0.748263 0.95374 0.838598 B-Route 466 286 123 0.619681 0.791172 0.695004 I-Negation 80 152 190 0.344828 0.296296 0.318725 B-Procedure_Name 1453 739 562 0.662865 0.721092 0.690754 I-Allergenic_substance 6 1 7 0.857143 0.461538 0.6 B-Allergenic_substance 74 31 23 0.704762 0.762887 0.732673 I-Weight 46 43 17 0.516854 0.730159 0.605263 B-Lab_Name 639 189 287 0.771739 0.690065 0.72862 I-Modifier 104 156 417 0.4 0.199616 0.266325 I-Temperature 2 7 13 0.222222 0.133333 0.166667 I-Drug_Name 334 237 290 0.584939 0.535256 0.558996 I-Lab_Name 271 157 140 0.633178 0.659367 0.646007 B-Respiratory_Rate 46 6 5 0.884615 0.901961 0.893204 Macro-average 37896 15237 14343 0.621144 0.562248 0.59023 Micro-average 37896 15237 14343 0.713229 0.725435 0.71928 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from jhoonk) author: John Snow Labs name: distilbert_qa_jhoonk_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `jhoonk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_jhoonk_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725579171.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_jhoonk_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725579171.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_jhoonk_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_jhoonk_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_jhoonk").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_jhoonk_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/jhoonk/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering model (from tucan9389) author: John Snow Labs name: distilbert_qa_tucan9389_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `tucan9389`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_tucan9389_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726443115.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_tucan9389_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726443115.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tucan9389_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tucan9389_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_tucan9389").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_tucan9389_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tucan9389/distilbert-base-uncased-finetuned-squad --- layout: model title: Detect Stock Markets in texts author: John Snow Labs name: finner_wiki_stockexchange date: 2023-01-15 tags: [stock, exchange, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model, aimed to detect Stock Exchanges / Stock Market names or abbreviations. It was trained with wikipedia texts about companies. ## Predicted Entities `STOCK_EXCHANGE`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_wiki_stockexchange_en_1.0.0_3.0_1673796187398.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_wiki_stockexchange_en_1.0.0_3.0_1673796187398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) chunks = finance.NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") ner = finance.NerModel().pretrained("finner_wiki_stockexchange", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") pipe = nlp.Pipeline(stages=[documenter, sentencizer, tokenizer, embeddings, ner, chunks]) model = pipe.fit(df) res = model.transform(df) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.begin, res.ner_chunk.end, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['3']['sentence']").alias("sentence_id"), F.expr("cols['0']").alias("chunk"), F.expr("cols['2']").alias("end"), F.expr("cols['3']['entity']").alias("ner_label"))\ .filter("ner_label!='O'")\ .show(truncate=False) ```

## Results ```bash +-----------+------+---+--------------+ |sentence_id|chunk |end|ner_label | +-----------+------+---+--------------+ |0 |NASDAQ|126|STOCK_EXCHANGE| +-----------+------+---+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_wiki_stockexchange| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.1 MB| ## References Wikipedia ## Benchmarking ```bash label tp fp fn prec rec f1 I-STOCK_EXCHANGE 21 0 0 1.0 1.0 1.0 B-STOCK_EXCHANGE 18 1 0 0.94736844 1.0 0.972973 Macro-average 39 1 0 0.9736842 1.0 0.9866667 Micro-average 39 1 0 0.975 1.0 0.98734176 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from ceshine) author: John Snow Labs name: t5_paraphrase_paws_msrp_opinosis date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-paraphrase-paws-msrp-opinosis` is a English model originally trained by `ceshine`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_paraphrase_paws_msrp_opinosis_en_4.3.0_3.0_1675125013995.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_paraphrase_paws_msrp_opinosis_en_4.3.0_3.0_1675125013995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_paraphrase_paws_msrp_opinosis","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_paraphrase_paws_msrp_opinosis","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_paraphrase_paws_msrp_opinosis| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|923.5 MB| ## References - https://huggingface.co/ceshine/t5-paraphrase-paws-msrp-opinosis - https://github.com/ceshine/finetuning-t5/tree/master/paraphrase --- layout: model title: ELMo Embeddings author: John Snow Labs name: elmo date: 2020-01-31 task: Embeddings language: en nav_key: models edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: ElmoEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Computes contextualized word representations using character-based word representations and bidirectional LSTMs. This model outputs fixed embeddings at each LSTM layer and a learnable aggregation of the 3 layers. * `word_emb`: the character-based word representations with shape [batch_size, max_length, 512]. * `lstm_outputs1`: the first LSTM hidden state with shape [batch_size, max_length, 1024]. * `lstm_outputs2`: the second LSTM hidden state with shape [batch_size, max_length, 1024]. * `elmo`: the weighted sum of the 3 layers, where the weights are trainable. This tensor has shape [batch_size, max_length, 1024]. The complex architecture achieves state of the art results on several benchmarks. Note that this is a very computationally expensive module compared to word embedding modules that only perform embedding lookups. The use of an accelerator is recommended. The details are described in the paper "[Deep contextualized word representations](https://arxiv.org/abs/1802.05365)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/elmo_en_2.4.0_2.4_1580488815299.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/elmo_en_2.4.0_2.4_1580488815299.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = ElmoEmbeddings.pretrained("elmo", "en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") \ .setPoolingLayer("elmo") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = ElmoEmbeddings.pretrained("elmo", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") .setPoolingLayer("elmo") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.elmo').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_elmo_embeddings token [0.6922717690467834, -0.32613131403923035, 0.2... I [-0.7348151206970215, -0.09645576030015945, -0... love [0.16370956599712372, 0.1217058077454567, -0.0... NLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|elmo| |Type:|embeddings| |Compatibility:|Spark NLP 2.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|512|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/elmo/3](https://tfhub.dev/google/elmo/3) --- layout: model title: English BertForQuestionAnswering model (from andi611) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-squad2-with-ner-Pwhatisthe-conll2003-with-neg-with-repeat` is a English model orginally trained by `andi611`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat_en_4.0.0_3.0_1654537369079.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat_en_4.0.0_3.0_1654537369079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_conll.bert.large_uncased_pwhatisthe.by_andi611").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pwhatisthe_conll2003_with_neg_with_repeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/andi611/bert-large-uncased-whole-word-masking-squad2-with-ner-Pwhatisthe-conll2003-with-neg-with-repeat --- layout: model title: Fast Neural Machine Translation Model from English to Greek Languages author: John Snow Labs name: opus_mt_en_grk date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, grk, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `grk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_grk_xx_2.7.0_2.4_1609163504692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_grk_xx_2.7.0_2.4_1609163504692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_grk| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from baru98) author: John Snow Labs name: distilbert_qa_baru98_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `baru98`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_baru98_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770148875.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_baru98_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770148875.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_baru98_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_baru98_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_baru98_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/baru98/distilbert-base-uncased-finetuned-squad --- layout: model title: Detect Clinical Entities in Romanian (Bert, Base, Cased) author: John Snow Labs name: ner_clinical_bert date: 2022-11-22 tags: [licensed, clinical, ro, ner, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract clinical entities from Romanian clinical texts. This model is trained using `bert_base_cased` embeddings. ## Predicted Entities `Measurements`, `Form`, `Symptom`, `Route`, `Procedure`, `Disease_Syndrome_Disorder`, `Score`, `Drug_Ingredient`, `Pulse`, `Frequency`, `Date`, `Body_Part`, `Drug_Brand_Name`, `Time`, `Direction`, `Medical_Device`, `Imaging_Technique`, `Test`, `Imaging_Findings`, `Imaging_Test`, `Test_Result`, `Weight`, `Clinical_Dept`, `Units` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_ro_4.2.2_3.0_1669124033852.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_ro_4.2.2_3.0_1669124033852.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------+-------------------------+ |chunks |entities | +--------------------------+-------------------------+ |Angio CT cardio-toracic |Imaging_Test | |Atrezie |Disease_Syndrome_Disorder| |valva pulmonara |Body_Part | |Hipoplazie |Disease_Syndrome_Disorder| |VS |Body_Part | |Atrezie |Disease_Syndrome_Disorder| |VAV stang |Body_Part | |Anastomoza Glenn |Disease_Syndrome_Disorder| |Tromboza |Disease_Syndrome_Disorder| |Sectia Clinica Cardiologie|Clinical_Dept | |GE Revolution HD |Medical_Device | |Branula albastra |Medical_Device | |membrului superior drept |Body_Part | |Scout |Body_Part | |30 ml |Dosage | |Iomeron 350 |Drug_Ingredient | |2.2 ml/s |Dosage | |20 ml |Dosage | |ser fiziologic |Drug_Ingredient | |angio-CT |Imaging_Test | +--------------------------+-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_bert| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.3 MB| ## Benchmarking ```bash label precision recall f1-score support Body_Part 0.91 0.93 0.92 679 Clinical_Dept 0.68 0.65 0.67 97 Date 0.99 0.99 0.99 87 Direction 0.66 0.76 0.70 50 Disease_Syndrome_Disorder 0.73 0.76 0.74 121 Dosage 0.78 1.00 0.87 38 Drug_Ingredient 0.90 0.94 0.92 48 Form 1.00 1.00 1.00 6 Imaging_Findings 0.86 0.82 0.84 201 Imaging_Technique 0.92 0.92 0.92 26 Imaging_Test 0.93 0.98 0.95 205 Measurements 0.71 0.69 0.70 214 Medical_Device 0.85 0.81 0.83 42 Pulse 0.82 1.00 0.90 9 Route 1.00 0.91 0.95 33 Score 1.00 0.98 0.99 41 Time 1.00 1.00 1.00 28 Units 0.60 0.93 0.73 88 Weight 0.82 1.00 0.90 9 micro-avg 0.84 0.87 0.86 2037 macro-avg 0.70 0.74 0.72 2037 weighted-avg 0.84 0.87 0.85 2037 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Original_PubMedBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Original-PubMedBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_PubMedBERT_384_en_4.0.0_3.0_1657108669858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Original_PubMedBERT_384_en_4.0.0_3.0_1657108669858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Original_PubMedBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Original-PubMedBERT-384 --- layout: model title: Mapping Diseases from the KEGG Database to Their Corresponding Categories, Descriptions and Clinical Vocabularies author: John Snow Labs name: kegg_disease_mapper date: 2022-11-18 tags: [disease, category, description, icd10, icd11, mesh, brite, en, clinical, chunk_mapper, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps diseases with their corresponding `category`, `description`, `icd10_code`, `icd11_code`, `mesh_code`, and hierarchical `brite_code`. This model was trained with the data from the KEGG database. ## Predicted Entities `category`, `description`, `icd10_code`, `icd11_code`, `mesh_code`, `brite_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/kegg_disease_mapper_en_4.2.2_3.0_1668794743905.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/kegg_disease_mapper_en_4.2.2_3.0_1668794743905.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained("ner_diseases", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ chunkerMapper = ChunkMapperModel.pretrained("kegg_disease_mapper", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["description", "category", "icd10_code", "icd11_code", "mesh_code", "brite_code"])\ pipeline = Pipeline().setStages([ document_assembler, sentence_detector, tokenizer, word_embeddings, ner, converter, chunkerMapper]) text= "A 55-year-old female with a history of myopia, kniest dysplasia and prostate cancer. She was on glipizide , and dapagliflozin for congenital nephrogenic diabetes insipidus." data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_diseases", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val chunkerMapper = ChunkMapperModel.pretrained("kegg_disease_mapper", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("mappings") .setRels(Array("description", "category", "icd10_code", "icd11_code", "mesh_code", "brite_code")) val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner, converter, chunkerMapper)) val text= "A 55-year-old female with a history of myopia, kniest dysplasia and prostate cancer. She was on glipizide , and dapagliflozin for congenital nephrogenic diabetes insipidus." val data = Seq(text).toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.kegg_disease").predict("""A 55-year-old female with a history of myopia, kniest dysplasia and prostate cancer. She was on glipizide , and dapagliflozin for congenital nephrogenic diabetes insipidus.""") ```

## Results ```bash +-----------------------------------------+--------------------------------------------------+-----------------------+----------+----------+---------+-----------------------+ | ner_chunk| description| category|icd10_code|icd11_code|mesh_code| brite_code| +-----------------------------------------+--------------------------------------------------+-----------------------+----------+----------+---------+-----------------------+ | myopia|Myopia is the most common ocular disorder world...| Nervous system disease| H52.1| 9D00.0| D009216| 08402,08403| | kniest dysplasia|Kniest dysplasia is an autosomal dominant chond...|Congenital malformation| Q77.7| LD24.3| C537207| 08402,08403| | prostate cancer|Prostate cancer constitutes a major health prob...| Cancer| C61| 2C82| NONE|08402,08403,08442,08441| |congenital nephrogenic diabetes insipidus|Nephrogenic diabetes insipidus (NDI) is charact...| Urinary system disease| N25.1| GB90.4A| D018500| 08402,08403| +-----------------------------------------+--------------------------------------------------+-----------------------+----------+----------+---------+-----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|kegg_disease_mapper| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|595.6 KB| --- layout: model title: Multilingual BertForQuestionAnswering model (from salti) author: John Snow Labs name: bert_qa_salti_bert_base_multilingual_cased_finetuned_squad date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-squad` is a Multilingual model orginally trained by `salti`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_salti_bert_base_multilingual_cased_finetuned_squad_xx_4.0.0_3.0_1654180181226.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_salti_bert_base_multilingual_cased_finetuned_squad_xx_4.0.0_3.0_1654180181226.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_salti_bert_base_multilingual_cased_finetuned_squad","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_salti_bert_base_multilingual_cased_finetuned_squad","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.squad.bert.multilingual_base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_salti_bert_base_multilingual_cased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/salti/bert-base-multilingual-cased-finetuned-squad - https://wandb.ai/salti/mBERT_QA/runs/wkqzhrp2 --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-10_H-512_A-8_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185208664.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185208664.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.uncased_10l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_10_H_512_A_8_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|178.3 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-10_H-512_A-8_squad2_covid-qna --- layout: model title: Turkish BertForQuestionAnswering model (from oguzhanolm) author: John Snow Labs name: bert_qa_loodos_bert_base_uncased_QA_fine_tuned date: 2022-06-02 tags: [tr, open_source, question_answering, bert] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `loodos-bert-base-uncased-QA-fine-tuned` is a Turkish model orginally trained by `oguzhanolm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_loodos_bert_base_uncased_QA_fine_tuned_tr_4.0.0_3.0_1654188187941.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_loodos_bert_base_uncased_QA_fine_tuned_tr_4.0.0_3.0_1654188187941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_loodos_bert_base_uncased_QA_fine_tuned","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_loodos_bert_base_uncased_QA_fine_tuned","tr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_loodos_bert_base_uncased_QA_fine_tuned| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tr| |Size:|412.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/oguzhanolm/loodos-bert-base-uncased-QA-fine-tuned - https://github.com/TQuad/turkish-nlp-qa-dataset - https://paperswithcode.com/sota?task=Question+Answering&dataset=TQuAD --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from navteca) author: John Snow Labs name: roberta_qa_navteca_large_squad2 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad2` is a English model originally trained by `navteca`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_navteca_large_squad2_en_4.2.4_3.0_1669988103804.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_navteca_large_squad2_en_4.2.4_3.0_1669988103804.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_navteca_large_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_navteca_large_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_navteca_large_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/navteca/roberta-large-squad2 - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: RE Pipeline between Dates and Clinical Entities author: John Snow Labs name: re_date_clinical_pipeline date: 2022-03-31 tags: [licensed, clinical, relation_extraction, date, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_date_clinical](https://nlp.johnsnowlabs.com/2021/01/18/re_date_clinical_en.html) model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb#scrollTo=d5wI11ptz7hi){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_date_clinical_pipeline_en_3.4.1_3.0_1648734471721.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_date_clinical_pipeline_en_3.4.1_3.0_1648734471721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("re_date_clinical_pipeline", "en", "clinical/models") pipeline.fullAnnotate("This 73 y/o patient had CT on 1/12/95, with progressive memory and cognitive decline since 8/11/94.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("re_date_clinical_pipeline", "en", "clinical/models") pipeline.fullAnnotate("This 73 y/o patient had CT on 1/12/95, with progressive memory and cognitive decline since 8/11/94.") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.date_clinical.pipeline").predict("""This 73 y/o patient had CT on 1/12/95, with progressive memory and cognitive decline since 8/11/94.""") ```

## Results ```bash | | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |---|-----------|---------|---------------|-------------|------------------------------------------|---------|-------------|-------------|---------|------------| | 0 | 1 | Test | 24 | 25 | CT | Date | 31 | 37 | 1/12/95 | 1.0 | | 1 | 1 | Symptom | 45 | 84 | progressive memory and cognitive decline | Date | 92 | 98 | 8/11/94 | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_date_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - PerceptronModel - DependencyParserModel - RelationExtractionModel --- layout: model title: Legal Shareholder Services Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_shareholder_services_agreement_bert date: 2023-01-26 tags: [en, legal, classification, shareholder, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_shareholder_services_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `shareholder-services-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `shareholder-services-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_shareholder_services_agreement_bert_en_1.0.0_3.0_1674735062182.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_shareholder_services_agreement_bert_en_1.0.0_3.0_1674735062182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_shareholder_services_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[shareholder-services-agreement]| |[other]| |[other]| |[shareholder-services-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_shareholder_services_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.99 0.99 116 shareholder-services-agreement 0.98 0.98 0.98 48 accuracy - - 0.99 164 macro-avg 0.99 0.99 0.99 164 weighted-avg 0.99 0.99 0.99 164 ``` --- layout: model title: Smaller BERT Embeddings (L-4_H-768_A-12) author: John Snow Labs name: small_bert_L4_768 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L4_768_en_2.6.0_2.4_1598345024690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L4_768_en_2.6.0_2.4_1598345024690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L4_768", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L4_768", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L4_768').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L4_768_embeddings I [0.14908359944820404, -0.06654861569404602, 0.... love [0.9139627814292908, 0.2444770336151123, 0.952... NLP [1.1467561721801758, -0.11340214312076569, 1.1... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L4_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1 --- layout: model title: Legal Maintenance Clause Binary Classifier author: John Snow Labs name: legclf_maintenance_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `maintenance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `maintenance` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_maintenance_clause_en_1.0.0_3.2_1660123713462.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_maintenance_clause_en_1.0.0_3.2_1660123713462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_maintenance_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[maintenance]| |[other]| |[other]| |[maintenance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_maintenance_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support maintenance 0.93 0.91 0.92 95 other 0.96 0.97 0.96 201 accuracy - - 0.95 296 macro-avg 0.95 0.94 0.94 296 weighted-avg 0.95 0.95 0.95 296 ``` --- layout: model title: Smaller BERT Sentence Embeddings (L-2_H-256_A-4) author: John Snow Labs name: sent_small_bert_L2_256 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L2_256_en_2.6.0_2.4_1598350372298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L2_256_en_2.6.0_2.4_1598350372298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L2_256", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L2_256", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L2_256').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L2_256_embeddings sentence [-1.0944892168045044, -1.8199821710586548, 1.4... I hate cancer [-0.8097536563873291, -1.0587245225906372, 1.2... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L2_256| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1 --- layout: model title: Swahili XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili date: 2022-08-01 tags: [sw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: sw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-wolof-finetuned-ner-swahili` is a Swahili model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili_sw_4.1.0_3.0_1659356044720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili_sw_4.1.0_3.0_1659356044720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili","sw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili","sw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_wolof_finetuned_ner_swahili| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|sw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-wolof-finetuned-ner-swahili - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Ukrainian Named Entity Recognition (from ukr-models) author: John Snow Labs name: xlmroberta_ner_uk_ner date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, uk, open_source] task: Named Entity Recognition language: uk edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `uk-ner` is a Ukrainian model orginally trained by `ukr-models`. ## Predicted Entities `LOC`, `ORG`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_uk_ner_uk_3.4.2_3.0_1652808897121.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_uk_ner_uk_3.4.2_3.0_1652808897121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_uk_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|uk| |Size:|406.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ukr-models/uk-ner - https://pypi.org/project/tokenize_uk/ --- layout: model title: English asr_liepa_lithuanian TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: pipeline_asr_liepa_lithuanian date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_liepa_lithuanian` is a English model originally trained by birgermoell. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_liepa_lithuanian_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_liepa_lithuanian_en_4.2.0_3.0_1664111880638.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_liepa_lithuanian_en_4.2.0_3.0_1664111880638.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_liepa_lithuanian', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_liepa_lithuanian", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_liepa_lithuanian| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|228.1 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Radiology Related Entities author: John Snow Labs name: ner_radiology date: 2021-01-18 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 2.7.0 spark_version: 2.4 tags: [en, ner, licensed, clinical] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for radiology related texts and reports. ## Predicted Entities `ImagingTest`, `Imaging_Technique`, `ImagingFindings`, `OtherFindings`, `BodyPart`, `Direction`, `Test`, `Symptom`, `Disease_Syndrome_Disorder`, `Medical_Device`, `Procedure`, `Measurements`, `Units` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_RADIOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_radiology_en_2.7.0_2.4_1610995075088.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_radiology_en_2.7.0_2.4_1610995075088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an NLP pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

## Results ```bash | | chunks | entities | |----|-----------------------|---------------------------| | 0 | Bilateral | Direction | | 1 | breast | BodyPart | | 2 | ultrasound | ImagingTest | | 3 | ovoid mass | ImagingFindings | | 4 | 0.5 x 0.5 x 0.4 | Measurements | | 5 | cm | Units | | 6 | anteromedial aspect | Direction | | 7 | left | Direction | | 8 | shoulder | BodyPart | | 9 | mass | ImagingFindings | | 10 | isoechoic echotexture | ImagingFindings | | 11 | muscle | BodyPart | | 12 | internal color flow | ImagingFindings | | 13 | benign fibrous tissue | ImagingFindings | | 14 | lipoma | Disease_Syndrome_Disorder | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_radiology| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on a custom dataset comprising of MIMIC-CXR and MT Radiology texts ## Benchmarking ```bash label tp fp fn total precision recall f1 OtherFindings 8.0 15.0 63.0 71.0 0.3478 0.1127 0.1702 Measurements 481.0 30.0 15.0 496.0 0.9413 0.9698 0.9553 Direction 650.0 137.0 94.0 744.0 0.8259 0.8737 0.8491 ImagingFindings 1345.0 355.0 324.0 1669.0 0.7912 0.8059 0.7985 BodyPart 1942.0 335.0 290.0 2232.0 0.8529 0.8701 0.8614 Medical_Device 236.0 75.0 64.0 300.0 0.7588 0.7867 0.7725 Test 222.0 41.0 48.0 270.0 0.8441 0.8222 0.833 Procedure 269.0 117.0 116.0 385.0 0.6969 0.6987 0.6978 ImagingTest 263.0 50.0 43.0 306.0 0.8403 0.8595 0.8498 Symptom 498.0 101.0 132.0 630.0 0.8314 0.7905 0.8104 Disease_Syndrome_... 1180.0 258.0 200.0 1380.0 0.8206 0.8551 0.8375 Units 269.0 10.0 2.0 271.0 0.9642 0.9926 0.9782 Imaging_Technique 140.0 38.0 25.0 165.0 0.7865 0.8485 0.8163 macro - - - - - - 0.7524 micro - - - - - - 0.8315 ``` --- layout: model title: Translate English to Kirundi Pipeline author: John Snow Labs name: translate_en_rn date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, rn, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `rn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_rn_xx_2.7.0_2.4_1609699240949.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_rn_xx_2.7.0_2.4_1609699240949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_rn", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_rn", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.rn').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_rn| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Hindi asr_Wav2Vec2_xls_r_300m_final TFWav2Vec2ForCTC from LegolasTheElf author: John Snow Labs name: asr_Wav2Vec2_xls_r_300m_final date: 2022-09-26 tags: [wav2vec2, hi, audio, open_source, asr] task: Automatic Speech Recognition language: hi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_xls_r_300m_final` is a Hindi model originally trained by LegolasTheElf. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Wav2Vec2_xls_r_300m_final_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_xls_r_300m_final_hi_4.2.0_3.0_1664193590997.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_xls_r_300m_final_hi_4.2.0_3.0_1664193590997.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Wav2Vec2_xls_r_300m_final", "hi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Wav2Vec2_xls_r_300m_final", "hi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Wav2Vec2_xls_r_300m_final| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|hi| |Size:|1.2 GB| --- layout: model title: Telugu Word Embeddings (DistilBERT) author: John Snow Labs name: distilbert_uncased date: 2021-12-14 tags: [open_source, embeddings, te, distilbert, telugu] task: Embeddings language: te edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a DistilBERT language model pre-trained on ~2 GB of the monolingual training corpus. The pre-training data was majorly taken from OSCAR. This model can be fine-tuned on various downstream tasks like text classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_uncased_te_3.1.0_3.0_1639472349482.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_uncased_te_3.1.0_3.0_1639472349482.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | Token | Embeddings | |---------- |---------------------------------------------------------------------- | | ఆంగ్ల | [1.3526772, 0.66792506, 0.80145407, -1.7625582, 1.222954, ...] | | పఠన | [2.0163336, -0.11855277, -0.71146804, -0.3959106, -0.63389313, ...] | | పేరాల | [1.0630535, 0.2587409, 0.09540532, -0.048794597, 1.0124478, ...] | | యొక్క | [1.4005142, 0.43655983, 0.5112152, -0.9843408, 0.9581941, ...] | | గొప్ప | [1.6955082, 0.40451798, 0.8449157, -1.0998198, 0.80302626, ...] | | మూలం | [2.0383484, 0.90390867, -0.52174926, -0.637539, 0.29188454, ...] | | కోసం | [1.3596793, 1.0218208, 0.26274702, -0.2437865, 0.50547075, ...] | | చూస్తున్నారా | [1.4825231, 0.6084269, 1.5597858, -1.0951629, 0.33125773, ...] | | ? | [2.86698, -0.07081009, 0.078133255, 0.43756652, 0.05295326, ...] | | మీరు | [1.0796824, 0.35925022, 0.51510495, -0.9841369, 0.39694318, ...] | | సరైన | [1.1148729, -0.004858747, 0.041157544, -0.5826167, 0.24176109, ...] | | స్థలానికి | [1.2047833, 0.119116426, -0.039619423, -0.48747823, 0.15061232, ...] | | వచ్చారు | [1.1785411, -0.013213344, 0.14526407, -0.60479, 0.031448614, ...] | | . | [0.76072985, -1.9430697, -1.6266187, 0.46296686, -2.2197602, ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_uncased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|te| |Size:|249.4 MB| |Case sensitive:|false| --- layout: model title: Side Effect Classification Pipeline - Voice of the Patient author: John Snow Labs name: bert_sequence_classifier_vop_side_effect_pipeline date: 2023-06-13 tags: [pipeline, classification, side_effect, vop, clinical, en, licensed] task: Text Classification language: en edition: Healthcare NLP 4.4.3 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline includes the Medical Bert for Sequence Classification model to classify health-related text in colloquial language according to the presence or absence of mentions of side effects. The pipeline is built on the top of [bert_sequence_classifier_vop_side_effect](https://nlp.johnsnowlabs.com/2023/05/24/bert_sequence_classifier_vop_side_effect_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_side_effect_pipeline_en_4.4.3_3.2_1686700519111.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_side_effect_pipeline_en_4.4.3_3.2_1686700519111.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use This pipeline includes the Medical Bert for Sequence Classification model to classify health-related text in colloquial language according to the presence or absence of mentions of side effects.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_sequence_classifier_vop_side_effect_pipeline", "en", "clinical/models") pipeline.annotate("I felt kind of dizzy after taking that medication for a month.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_sequence_classifier_vop_side_effect_pipeline", "en", "clinical/models") val result = pipeline.annotate(I felt kind of dizzy after taking that medication for a month.) ```

## Results ```bash | text | prediction | |:---------------------------------------------------------------|:-------------| | I felt kind of dizzy after taking that medication for a month. | True | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_side_effect_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|406.4 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForSequenceClassification --- layout: model title: Legal Successors Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_successors_bert date: 2023-03-05 tags: [en, legal, classification, clauses, successors, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Successors` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Successors`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_successors_bert_en_1.0.0_3.0_1678049877265.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_successors_bert_en_1.0.0_3.0_1678049877265.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_successors_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Successors]| |[Other]| |[Other]| |[Successors]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_successors_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 1.00 1.00 1.0 76 Successors 1.00 1.00 1.0 55 accuracy - - 1.0 131 macro-avg 1.00 1.00 1.0 131 weighted-avg 1.00 1.00 1.0 131 ``` --- layout: model title: English image_classifier_vit_dog_vs_chicken ViTForImageClassification from lewtun author: John Snow Labs name: image_classifier_vit_dog_vs_chicken date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_dog_vs_chicken` is a English model originally trained by lewtun. ## Predicted Entities `crispy fried chicken`, `poodle` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_vs_chicken_en_4.1.0_3.0_1660167393770.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_vs_chicken_en_4.1.0_3.0_1660167393770.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_dog_vs_chicken", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_dog_vs_chicken", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_dog_vs_chicken| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForQuestionAnswering Cased model (from Ahmed007) author: John Snow Labs name: distilbert_qa_close_book date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `close-book` is a English model originally trained by `Ahmed007`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_close_book_en_4.3.0_3.0_1672765961995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_close_book_en_4.3.0_3.0_1672765961995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_close_book","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_close_book","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_close_book| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Ahmed007/close-book --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_0 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-0` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_0_en_4.3.0_3.0_1672767389948.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_0_en_4.3.0_3.0_1672767389948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-0 --- layout: model title: Translate Afro-Asiatic languages to English Pipeline author: John Snow Labs name: translate_afa_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, afa, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `afa` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_afa_en_xx_2.7.0_2.4_1609698877865.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_afa_en_xx_2.7.0_2.4_1609698877865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_afa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_afa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.afa.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_afa_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Extract Clinical Entities from Voice of the Patient Documents (embeddings_clinical_large) author: John Snow Labs name: ner_vop_emb_clinical_large date: 2023-06-06 tags: [licensed, clinical, en, ner, vop, patient] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `Gender`, `Employment`, `BodyPart`, `Vaccine`, `Age`, `PsychologicalCondition`, `Form`, `Substance`, `ClinicalDept`, `Drug`, `AdmissionDischarge`, `DateTime`, `Test`, `Laterality`, `Route`, `Disease`, `VitalTest`, `Dosage`, `Duration`, `Frequency`, `Allergen`, `Symptom`, `Procedure`, `RelationshipStatus`, `HealthStatus`, `Modifier`, `MedicalDevice`, `SubstanceQuantity`, `InjuryOrPoisoning`, `TestResult`, `Treatment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_large_en_4.4.3_3.0_1686073213251.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_large_en_4.4.3_3.0_1686073213251.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") glove_embeddings = WordEmbeddingsModel.pretrained('glove_100d') \ .setInputCols(['sentence', 'token']) \ .setOutputCol('embeddings') deid_ner = MedicalNerModel.pretrained("ner_deid_generic_glove", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, glove_embeddings, deid_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame(pd.DataFrame({"text": ["""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227."""]}))) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val glove_embeddings = WordEmbeddingsModel.pretrained("glove_100d") .setInputCols(Array("sentence', 'token")) .setOutputCol("embeddings") val deid_ner = MedicalNerModel.pretrained("ner_deid_generic_glove", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val nlpPipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, glove_embeddings, deid_ner, ner_converter)) val result = nlpPipeline.fit(Seq.empty["A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227."].toDS.toDF("text")).transform(data) ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |2093-01-13 |DATE | |David Hale |NAME | |Hendrickson, Ora |NAME | |7194334 |ID | |01/13/93 |DATE | |Oliveira |NAME | |25 |AGE | |1-11-2000 |DATE | |Cocke County Baptist Hospital|LOCATION | |0295 Keats Street |LOCATION | |(302) 786-5227 |CONTACT | +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_glove| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source A custom data set which is created from the i2b2-PHI train and the augmented version of the i2b2-PHI train set is used. --- layout: model title: German BertForTokenClassification Cased model (from Sahajtomar) author: John Snow Labs name: bert_token_classifier_ner_legal date: 2022-11-30 tags: [de, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `NER_legal_de` is a German model originally trained by `Sahajtomar`. ## Predicted Entities `UN`, `ST`, `VO`, `VS`, `ORG`, `STR`, `RS`, `LIT`, `PER`, `GRT`, `LD`, `INN`, `VT`, `AN`, `EUN`, `MRK`, `RR`, `GS`, `LDS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ner_legal_de_4.2.4_3.0_1669814042794.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ner_legal_de_4.2.4_3.0_1669814042794.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_legal","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_legal","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|de| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Sahajtomar/NER_legal_de --- layout: model title: Medical Question Answering (biogpt) author: John Snow Labs name: medical_qa_biogpt date: 2023-03-09 tags: [licensed, clinical, en, gpt, biogpt, pubmed, question_answering, tensorflow] task: Question Answering language: en edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true recommended: true engine: tensorflow annotator: MedicalQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is directly ported from the official BioGPT [implementation](https://github.com/microsoft/BioGPT) that is trained on Pubmed abstracts and then finetuned with PubmedQA dataset. It is the baseline version called [BioGPT-QA-PubMedQA-BioGPT](https://msramllasc.blob.core.windows.net/modelrelease/BioGPT/checkpoints/QA-PubMedQA-BioGPT.tgz). It can generate two types of answers, short and long. Types of questions are supported: `"short"`(producing yes/no/maybe) answers and `"full"` (long answers). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/BIOGPT_MEDICAL_QUESTION_ANSWERING/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/31.Medical_Question_Answering.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/medical_qa_biogpt_en_4.3.1_3.0_1678355315206.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/medical_qa_biogpt_en_4.3.1_3.0_1678355315206.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler()\ .setInputCols("question", "context")\ .setOutputCols("document_question", "document_context") med_qa = sparknlp_jsl.annotators.MedicalQuestionAnswering\ .pretrained("medical_qa_biogpt","en","clinical/models")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setMaxNewTokens(30)\ .setTopK(1)\ .setQuestionType("long") # "short" pipeline = Pipeline(stages=[document_assembler, med_qa]) paper_abstract = "The visual indexing theory proposed by Zenon Pylyshyn (Cognition, 32, 65–97, 1989) predicts that visual attention mechanisms are employed when mental images are projected onto a visual scene. Recent eye-tracking studies have supported this hypothesis by showing that people tend to look at empty places where requested information has been previously presented. However, it has remained unclear to what extent this behavior is related to memory performance. The aim of the present study was to explore whether the manipulation of spatial attention can facilitate memory retrieval. In two experiments, participants were asked first to memorize a set of four objects and then to determine whether a probe word referred to any of the objects. The results of both experiments indicate that memory accuracy is not affected by the current focus of attention and that all the effects of directing attention to specific locations on response times can be explained in terms of stimulus–stimulus and stimulus–response spatial compatibility." long_question = "What is the effect of directing attention on memory?" yes_no_question = "Does directing attention improve memory for items?" data = spark.createDataFrame( [ [long_question, paper_abstract, "long"], [yes_no_question, paper_abstract, "short"], ] ).toDF("question", "context", "question_type") pipeline.fit(data).transform(data.where("question_type == 'long'"))\ .select("answer.result")\ .show(truncate=False) pipeline.fit(data).transform(data.where("question_type == 'short'"))\ .select("answer.result")\ .show(truncate=False) ``` ```scala val document_assembler = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val med_qa = MedicalQuestionAnswering .pretrained("medical_qa_biogpt", "en", "clinical/models") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setMaxNewTokens(30) .setTopK(1) .setQuestionType("long") # "short" val pipeline = new Pipeline().setStages(Array(document_assembler, med_qa)) paper_abstract = "The visual indexing theory proposed by Zenon Pylyshyn (Cognition, 32, 65–97, 1989) predicts that visual attention mechanisms are employed when mental images are projected onto a visual scene. Recent eye-tracking studies have supported this hypothesis by showing that people tend to look at empty places where requested information has been previously presented. However, it has remained unclear to what extent this behavior is related to memory performance. The aim of the present study was to explore whether the manipulation of spatial attention can facilitate memory retrieval. In two experiments, participants were asked first to memorize a set of four objects and then to determine whether a probe word referred to any of the objects. The results of both experiments indicate that memory accuracy is not affected by the current focus of attention and that all the effects of directing attention to specific locations on response times can be explained in terms of stimulus–stimulus and stimulus–response spatial compatibility." long_question = "What is the effect of directing attention on memory?" yes_no_question = "Does directing attention improve memory for items?" val data = Seq( (long_question, paper_abstract,"long" ), (yes_no_question, paper_abstract, "short")) .toDS.toDF("question", "context", "question_type") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_biomedical_bc2gm_pipeline", "en", "clinical/models") result = pipeline.fullAnnotate("""Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_biomedical_bc2gm_pipeline", "en", "clinical/models") val result = pipeline.fullAnnotate("""Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biomedical_bc2gm.pipeline").predict("""Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections.""") ```

## Results ```bash +-----------+------------+ |chunk |ner_label | +-----------+------------+ |S-100 |GENE_PROTEIN| |HMB-45 |GENE_PROTEIN| |cytokeratin|GENE_PROTEIN| +-----------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_biomedical_bc2gm_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Japanese BertForQuestionAnswering Base model (from KoichiYasuoka) author: John Snow Labs name: bert_qa_base_japanese_wikipedia_ud_head date: 2022-06-28 tags: [ja, open_source, bert, question_answering] task: Question Answering language: ja edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-japanese-wikipedia-ud-head` is a Japanese model originally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_japanese_wikipedia_ud_head_ja_4.0.0_3.0_1656413487634.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_japanese_wikipedia_ud_head_ja_4.0.0_3.0_1656413487634.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_japanese_wikipedia_ud_head","ja") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_japanese_wikipedia_ud_head","ja") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ja.answer_question.wikipedia.bert.base").predict("""私の名前は何ですか？|||"私の名前はクララで、私はバークレーに住んでいます。""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_japanese_wikipedia_ud_head| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ja| |Size:|338.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KoichiYasuoka/bert-base-japanese-wikipedia-ud-head - https://github.com/UniversalDependencies/UD_Japanese-GSDLUW --- layout: model title: Pipeline to Detect Cellular/Molecular Biology Entities author: John Snow Labs name: ner_cellular_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_cellular](https://nlp.johnsnowlabs.com/2021/03/31/ner_cellular_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cellular_pipeline_en_3.4.1_3.0_1647870149220.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cellular_pipeline_en_3.4.1_3.0_1647870149220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_cellular_pipeline", "en", "clinical/models") pipeline.annotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` ```scala val pipeline = new PretrainedPipeline("ner_cellular_pipeline", "en", "clinical/models") pipeline.annotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cellular.pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash |chunk |ner | +-----------------------------------------------------------+---------+ |intracellular signaling proteins |protein | |human T-cell leukemia virus type 1 promoter |DNA | |Tax |protein | |Tax-responsive element 1 |DNA | |cyclic AMP-responsive members |protein | |CREB/ATF family |protein | |transcription factors |protein | |Tax |protein | |human T-cell leukemia virus type 1 Tax-responsive element 1|DNA | |TRE-1), |DNA | |lacZ gene |DNA | |CYC1 promoter |DNA | |TRE-1 |DNA | |cyclic AMP response element-binding protein |protein | |CREB |protein | |CREB |protein | |GAL4 activation domain |protein | |GAD |protein | |reporter gene |DNA | |Tax |protein | +-----------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English BertForTokenClassification Uncased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_Original_scibert_scivocab_uncased date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4-Original-scibert_scivocab_uncased` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Original_scibert_scivocab_uncased_en_4.0.0_3.0_1657108142723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Original_scibert_scivocab_uncased_en_4.0.0_3.0_1657108142723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_Original_scibert_scivocab_uncased| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.4 MB| |Case sensitive:|false| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4-Original-scibert_scivocab_uncased --- layout: model title: Hindi RobertaForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: roberta_embeddings_indic_transformers date: 2022-12-12 tags: [hi, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: hi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-hi-roberta` is a Hindi model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_hi_4.2.4_3.0_1670858655317.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_hi_4.2.4_3.0_1670858655317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indic_transformers","hi") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indic_transformers","hi") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indic_transformers| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hi| |Size:|313.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-hi-roberta - https://oscar-corpus.com/ --- layout: model title: Korean ElectraForQuestionAnswering model (from monologg) author: John Snow Labs name: electra_qa_base_v2_finetuned_korquad date: 2022-06-22 tags: [ko, open_source, electra, question_answering] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `koelectra-base-v2-finetuned-korquad` is a Korean model originally trained by `monologg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_v2_finetuned_korquad_ko_4.0.0_3.0_1655922101194.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_v2_finetuned_korquad_ko_4.0.0_3.0_1655922101194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_v2_finetuned_korquad","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_v2_finetuned_korquad","ko") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.korquad.electra.base_v2.by_monologg").predict("""내 이름은 무엇입니까?|||"제 이름은 클라라이고 저는 버클리에 살고 있습니다.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_v2_finetuned_korquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|412.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/monologg/koelectra-base-v2-finetuned-korquad --- layout: model title: Company Name Normalization to Edgar Database author: John Snow Labs name: legel_edgar_company_name date: 2022-08-30 tags: [en, legal, companies, edgar, licensed] task: Entity Resolution language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an Entity Linking / Entity Resolution model, which allows you to map an extracted Company Name from any NER model, to the name used by SEC in Edgar Database. This can come in handy to afterwards use Edgar Chunk Mappers with the output of this resolution, to carry out data augmentation and retrieve additional information stored in Edgar Database about a company. For more information about data augmentation, check `Chunk Mapping` task in Models Hub. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/ER_EDGAR_CRUNCHBASE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legel_edgar_company_name_en_1.0.0_3.2_1661866264706.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legel_edgar_company_name_en_1.0.0_3.2_1661866264706.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk") \ .setOutputCol("sentence_embeddings") resolver = legal.SentenceEntityResolverModel.pretrained("legel_edgar_company_name", "en", "legal/models")\ .setInputCols(["ner_chunk", "sentence_embeddings"]) \ .setOutputCol("irs_code")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = nlp..Pipeline( stages = [ documentAssembler, embeddings, resolver]) lp = LightPipeline(pipelineModel) lp.fullAnnotate("CONTACT GOLD") ```

## Results ```bash +--------------+----------+---------------------------------------------------------+--------------------------------------------------------------------------------------------+-------------------------------------------+ | chunk | code | all_codes | resolutions | all_distances | +--------------+----------+---------------------------------------------------------+--------------------------------------------------------------------------------------------+-------------------------------------------+ | CONTACT GOLD | 981369960| [981369960, 271989147, 208531222, 273566922, 270348508] |[Contact Gold Corp, Guskin Gold Corp, Yinfu Gold Corp, MAGELLAN GOLD Corp, Star Gold Corp] | [0.1733, 0.3700, 0.3867, 0.4103, 0.4121] | +--------------+----------+---------------------------------------------------------+--------------------------------------------------------------------------------------------+-------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legel_edgar_company_name| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[original_company_name]| |Language:|en| |Size:|315.1 MB| |Case sensitive:|false| ## References In-house scrapping and postprocessing of SEC Edgar Database --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_dm256 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-dm256` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_dm256_en_4.3.0_3.0_1675115135612.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_dm256_en_4.3.0_3.0_1675115135612.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_dm256","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_dm256","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_dm256| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|370.2 MB| ## References - https://huggingface.co/google/t5-efficient-large-dm256 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Mapping SNOMED Codes with Their Corresponding ICD10-CM Codes author: John Snow Labs name: snomed_icd10cm_mapper date: 2022-06-26 tags: [clinical, licensed, icd10cm, chunk_mapper, en, snomed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps SNOMED codes to corresponding ICD10-CM codes. ## Predicted Entities `icd10cm_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapper_en_3.5.3_3.0_1656266755041.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapper_en_3.5.3_3.0_1656266755041.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_uncased", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("snomed_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel.pretrained("snomed_icd10cm_mapper", "en", "clinical/models")\ .setInputCols(["snomed_code"])\ .setOutputCol("icd10cm_mappings")\ .setRels(["icd10cm_code"]) pipeline = Pipeline( stages = [ documentAssembler, sbert_embedder, snomed_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline= LightPipeline(model) result = light_pipeline.fullAnnotate("Radiating chest pain") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbert_jsl_medium_uncased", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel.pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("snomed_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel.pretrained("snomed_icd10cm_mapper", "en", "clinical/models") .setInputCols("snomed_code") .setOutputCol("icd10cm_mappings") .setRels(Array("icd10cm_code")) val pipeline = new Pipeline(stages = Array( documentAssembler, sbert_embedder, snomed_resolver, chunkerMapper)) val data = Seq("Radiating chest pain").toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.snomed_to_icd10cm").predict("""Radiating chest pain""") ```

## Results ```bash | | ner_chunk | snomed_code | icd10cm_mappings | |---:|:---------------------|--------------:|:-------------------| | 0 | Radiating chest pain | 10000006 | R07.9 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_icd10cm_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[snomed_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|1.5 MB| ## References This pretrained model maps SNOMED codes to corresponding ICD10-CM codes under the Unified Medical Language System (UMLS). --- layout: model title: Pipeline to Summarize Clinical Notes author: John Snow Labs name: summarizer_clinical_jsl_pipeline date: 2023-05-31 tags: [licensed, en, clinical, summarization] task: Summarization language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_jsl](https://nlp.johnsnowlabs.com/2023/03/25/summarizer_clinical_jsl.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_pipeline_en_4.4.1_3.0_1685534653791.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_pipeline_en_4.4.1_3.0_1685534653791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_jsl_pipeline", "en", "clinical/models") text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_jsl_pipeline", "en", "clinical/models") val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash +--------+ |result | +--------+ |[melhor]| +--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|pt| |Size:|2.5 KB| --- layout: model title: Sentence Entity Resolver for HGNC author: John Snow Labs name: sbiobertresolve_hgnc date: 2023-03-26 tags: [hgnc, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted gene names and their short-form abbreviations to HGNC codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it returns the locus groups and locus types of the genes as aux labels separated by || under the metadata. ## Predicted Entities `HGNC Codes`, `Locus Group`, `Locus Type` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_hgnc_en_4.3.2_3.0_1679847290330.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_hgnc_en_4.3.2_3.0_1679847290330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | sent_id | ner_chunk | entity | HGNC Code | all_codes | resolutions | AUX | |----------:|:------------|:---------|:-------------|:------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------| | 0 | DUX4L20 | GENE | HGNC:50801 | ['HGNC:50801', 'HGNC:31982', 'HGNC:42423', 'HGNC:39776', 'HGNC:42023'...| ['DUX4L20 [double homeobox 4 like 20 (pseudogene)]', 'ANKRD20A4P [ankyrin repeat domain 20 family member A4, pseudogene]', ...| [pseudogene :: pseudogene, pseudogene :: pseudogene, non-coding RNA :: RNA, long non-coding, pseudogene :: pseudogene...| | 0 | FBXO48 | GENE | HGNC:33857 | ['HGNC:33857', 'HGNC:4930', 'HGNC:16653', 'HGNC:13114', 'HGNC:23535' ...| ['FBXO48 [F-box protein 48]', 'ZBTB48 [zinc finger and BTB domain containing 48]', 'MRPL48 [mitochondrial ribosomal protein' ...| [protein-coding gene :: gene with protein product, protein-coding gene :: gene with protein product, protein-coding gene...| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_hgnc| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[hgnc_code]| |Language:|en| |Size:|251.9 MB| |Case sensitive:|false| ## References https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/ --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10_en_4.3.0_3.0_1674213671532.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10_en_4.3.0_3.0_1674213671532.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_128_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|423.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-10 --- layout: model title: Spanish Named Entity Recognition (from MMG) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_large_ner_spanish date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, es, open_source] task: Named Entity Recognition language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-large-ner-spanish` is a Spanish model orginally trained by `MMG`. ## Predicted Entities `LOC`, `ORG`, `MISC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_large_ner_spanish_es_3.4.2_3.0_1652807519626.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_large_ner_spanish_es_3.4.2_3.0_1652807519626.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_large_ner_spanish| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/MMG/xlm-roberta-large-ner-spanish --- layout: model title: Translate English to Kongo Pipeline author: John Snow Labs name: translate_en_kg date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, kg, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `kg` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_kg_xx_2.7.0_2.4_1609687119024.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_kg_xx_2.7.0_2.4_1609687119024.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_kg", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_kg", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.kg').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_kg| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_10k_voxpopuli TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_base_10k_voxpopuli date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_10k_voxpopuli` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_10k_voxpopuli_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_10k_voxpopuli_en_4.2.0_3.0_1664022512444.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_10k_voxpopuli_en_4.2.0_3.0_1664022512444.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_10k_voxpopuli", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_10k_voxpopuli", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_10k_voxpopuli| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: English BertForQuestionAnswering model (from TingChenChang) author: John Snow Labs name: bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-cased-finetuned-xquadv1-finetuned-squad-colab` is a English model orginally trained by `TingChenChang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab_en_4.0.0_3.0_1654184544770.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab_en_4.0.0_3.0_1654184544770.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xquad_squad.bert.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_multi_cased_finetuned_xquadv1_finetuned_squad_colab| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/TingChenChang/bert-multi-cased-finetuned-xquadv1-finetuned-squad-colab --- layout: model title: Detect Anatomical Regions (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_anatomy date: 2021-09-30 tags: [anatomy, bertfortokenclassification, en, licensed, ner] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.2.2 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for anatomy terms. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `Anatomical_system`, `Cell`, `Cellular_component`, `Developing_anatomical_structure`, `Immaterial_anatomical_entity`, `Multi-tissue_structure`, `Organ`, `Organism_subdivision`, `Organism_substance`, `Pathological_formation`, `Tissue` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_en_3.2.2_2.4_1632991802389.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatomy_en_3.2.2_2.4_1632991802389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentenceDetector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_anatomy", "en", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("ner")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) pp_model = pipeline.fit(spark.createDataFrame(pd.DataFrame({'text': ['']}))) test_sentence = """This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""" result = pp_model.transform(spark.createDataFrame(pd.DataFrame({'text': [test_sentence]}))) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_anatomy", "en", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("ner") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("document","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentence_detector, tokenizer, tokenClassifier, ner_converter)) val data = Seq("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_anatomy").predict("""This is an 11-year-old female who comes in for two different things. 1. She was seen by the allergist. No allergies present, so she stopped her Allegra, but she is still real congested and does a lot of snorting. They do not notice a lot of snoring at night though, but she seems to be always like that. 2. On her right great toe, she has got some redness and erythema. Her skin is kind of peeling a little bit, but it has been like that for about a week and a half now.\nGeneral: Well-developed female, in no acute distress, afebrile.\nHEENT: Sclerae and conjunctivae clear. Extraocular muscles intact. TMs clear. Nares patent. A little bit of swelling of the turbinates on the left. Oropharynx is essentially clear. Mucous membranes are moist.\nNeck: No lymphadenopathy.\nChest: Clear.\nAbdomen: Positive bowel sounds and soft.\nDermatologic: She has got redness along her right great toe, but no bleeding or oozing. Some dryness of her skin. Her toenails themselves are very short and even on her left foot and her left great toe the toenails are very short.""") ```

## Results ```bash +-------------------+----------------------+ |chunk |ner_label | +-------------------+----------------------+ |great toe |Multi-tissue_structure| |skin |Organ | |conjunctivae |Multi-tissue_structure| |Extraocular muscles|Multi-tissue_structure| |Nares |Multi-tissue_structure| |turbinates |Multi-tissue_structure| |Oropharynx |Multi-tissue_structure| |Mucous membranes |Tissue | |Neck |Organism_subdivision | |bowel |Organ | |great toe |Multi-tissue_structure| |skin |Organ | |toenails |Organism_subdivision | |foot |Organism_subdivision | |great toe |Multi-tissue_structure| |toenails |Organism_subdivision | +-------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_anatomy| |Compatibility:|Healthcare NLP 3.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|512| ## Data Source Trained on the Anatomical Entity Mention (AnEM) corpus with 'embeddings_clinical'. http://www.nactem.ac.uk/anatomy/ ## Benchmarking ```bash label precision recall f1-score support B-Anatomical_system 1.00 0.50 0.67 4 B-Cell 0.89 0.96 0.92 74 B-Cellular_component 0.97 0.81 0.88 36 B-Developing_anatomical_structure 1.00 0.50 0.67 6 B-Immaterial_anatomical_entity 0.60 0.75 0.67 4 B-Multi-tissue_structure 0.75 0.86 0.80 58 B-Organ 0.86 0.88 0.87 48 B-Organism_subdivision 0.62 0.42 0.50 12 B-Organism_substance 0.89 0.81 0.85 31 B-Pathological_formation 0.91 0.91 0.91 32 B-Tissue 0.94 0.76 0.84 21 I-Anatomical_system 1.00 1.00 1.00 1 I-Cell 1.00 0.84 0.91 62 I-Cellular_component 0.92 0.85 0.88 13 I-Developing_anatomical_structure 1.00 1.00 1.00 1 I-Immaterial_anatomical_entity 1.00 1.00 1.00 1 I-Multi-tissue_structure 1.00 0.77 0.87 26 I-Organ 0.80 0.80 0.80 5 I-Organism_substance 1.00 0.71 0.83 7 I-Pathological_formation 1.00 0.94 0.97 16 I-Tissue 0.93 0.93 0.93 15 accuracy - - 0.84 473 macro-avg 0.87 0.77 0.83 473 weighted-avg 0.90 0.84 0.86 473 ``` --- layout: model title: Legal Remedies Clause Binary Classifier author: John Snow Labs name: legclf_remedies_clause date: 2023-02-13 tags: [en, legal, classification, remedies, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `remedies` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `remedies`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_remedies_clause_en_1.0.0_3.0_1676303332118.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_remedies_clause_en_1.0.0_3.0_1676303332118.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_remedies_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[remedies]| |[other]| |[other]| |[remedies]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_remedies_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 10 remedies 1.00 1.00 1.00 19 accuracy - - 1.00 29 macro-avg 1.00 1.00 1.00 29 weighted-avg 1.00 1.00 1.00 29 ``` --- layout: model title: Legal Form Of Indemnification Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_form_of_indemnification_agreement_bert date: 2023-01-26 tags: [en, legal, classification, indemnification, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_form_of_indemnification_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `form-of-indemnification-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `form-of-indemnification-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_form_of_indemnification_agreement_bert_en_1.0.0_3.0_1674732486525.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_form_of_indemnification_agreement_bert_en_1.0.0_3.0_1674732486525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_form_of_indemnification_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[form-of-indemnification-agreement]| |[other]| |[other]| |[form-of-indemnification-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_form_of_indemnification_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support form-of-indemnification-agreement 1.00 0.97 0.99 36 other 0.98 1.00 0.99 65 accuracy - - 0.99 101 macro-avg 0.99 0.99 0.99 101 weighted-avg 0.99 0.99 0.99 101 ``` --- layout: model title: English ElectraForQuestionAnswering model (from PremalMatalia) author: John Snow Labs name: electra_qa_base_best_squad2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-best-squad2` is a English model originally trained by `PremalMatalia`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_best_squad2_en_4.0.0_3.0_1655920339398.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_best_squad2_en_4.0.0_3.0_1655920339398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_best_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_best_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.electra.base").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_best_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/PremalMatalia/electra-base-best-squad2 --- layout: model title: Catalan, Valencian asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly TFWav2Vec2ForCTC from ccoreilly author: John Snow Labs name: pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly date: 2022-09-24 tags: [wav2vec2, ca, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ca edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly` is a Catalan, Valencian model originally trained by ccoreilly. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly_ca_4.2.0_3.0_1664035897662.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly_ca_4.2.0_3.0_1664035897662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly', lang = 'ca') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly", lang = "ca") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_100k_voxpopuli_catala_by_ccoreilly| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ca| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Russian (WikiNER 6B 300) author: John Snow Labs name: wikiner_6B_300 date: 2020-03-16 task: Named Entity Recognition language: ru edition: Spark NLP 2.4.4 spark_version: 2.4 tags: [ner, ru, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 6B 300 is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_RU){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_RU.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_ru_2.4.4_2.4_1584014001694.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_ru_2.4.4_2.4_1584014001694.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_6B_300", lang="xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = NerDLModel.pretrained("wikiner_6B_300", "ru") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.']], ["text"])) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_6B_300", lang="xx") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner_model = NerDLModel.pretrained("wikiner_6B_300", "ru") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Уильям Генри Гейтс III (родился 28 октября 1955 года) - американский бизнес-магнат, разработчик программного обеспечения, инвестор и филантроп. Он наиболее известен как соучредитель корпорации Microsoft. За время своей карьеры в Microsoft Гейтс занимал должности председателя, главного исполнительного директора (CEO), президента и главного архитектора программного обеспечения, а также был крупнейшим индивидуальным акционером до мая 2014 года. Он является одним из самых известных предпринимателей и пионеров микрокомпьютерная революция 1970-х и 1980-х годов. Гейтс родился и вырос в Сиэтле, штат Вашингтон, в 1975 году вместе с другом детства Полом Алленом в Альбукерке, штат Нью-Мексико, и основал компанию Microsoft. она стала крупнейшей в мире компанией-разработчиком программного обеспечения для персональных компьютеров. Гейтс руководил компанией в качестве председателя и генерального директора, пока в январе 2000 года не ушел с поста генерального директора, но остался председателем и стал главным архитектором программного обеспечения. В конце 1990-х Гейтс подвергся критике за свою деловую тактику, которая считалась антиконкурентной. Это мнение было подтверждено многочисленными судебными решениями. В июне 2006 года Гейтс объявил, что перейдет на неполный рабочий день в Microsoft и будет работать на полную ставку в Фонде Билла и Мелинды Гейтс, частном благотворительном фонде, который он и его жена Мелинда Гейтс создали в 2000 году. [ 9] Постепенно он передал свои обязанности Рэю Оззи и Крейгу Манди. Он ушел с поста президента Microsoft в феврале 2014 года и занял новую должность консультанта по технологиям для поддержки вновь назначенного генерального директора Сатья Наделла."""] ner_df = nlu.load('ru.ner.wikiner.glove.6B_300').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Уильям Генри Гейтс III|PER | |Microsoft |ORG | |Microsoft Гейтс |ORG | |CEO |ORG | |Гейтс |PER | |Сиэтле |LOC | |Вашингтон |LOC | |Полом Алленом |PER | |Альбукерке |LOC | |Нью-Мексико |LOC | |Microsoft |ORG | |Гейтс |PER | |Гейтс |PER | |Это мнение |MISC | |Гейтс |PER | |Microsoft |ORG | |Фонде Билла |MISC | |Мелинды Гейтс |PER | |Мелинда Гейтс |PER | |Постепенно |PER | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ru| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://ru.wikipedia.org](https://ru.wikipedia.org) --- layout: model title: English BertForTokenClassification Cased model (from testing) author: John Snow Labs name: bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711 date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-ingredient_sentiment_analysis-19126711` is a English model originally trained by `testing`. ## Predicted Entities `I`, `B` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711_en_4.2.4_3.0_1669814269719.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711_en_4.2.4_3.0_1669814269719.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autonlp_ingredient_sentiment_analysis_19126711| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/testing/autonlp-ingredient_sentiment_analysis-19126711 --- layout: model title: Fast Neural Machine Translation Model from Marshallese to English author: John Snow Labs name: opus_mt_mh_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, mh, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `mh` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_mh_en_xx_2.7.0_2.4_1609170065092.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_mh_en_xx_2.7.0_2.4_1609170065092.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_mh_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Ukrainian Part of Speech Tagger (Base) author: John Snow Labs name: bert_pos_bert_base_slavic_cyrillic_upos date: 2022-04-26 tags: [bert, pos, part_of_speech, uk, open_source] task: Part of Speech Tagging language: uk edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-slavic-cyrillic-upos` is a Ukrainian model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_slavic_cyrillic_upos_uk_3.4.2_3.0_1650993899127.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_slavic_cyrillic_upos_uk_3.4.2_3.0_1650993899127.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_slavic_cyrillic_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|uk| |Size:|668.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-base-slavic-cyrillic-upos - https://universaldependencies.org/be/ - https://universaldependencies.org/bg/ - https://universaldependencies.org/ru/ - https://universaldependencies.org/treebanks/sr_set/ - https://universaldependencies.org/treebanks/uk_iu/ - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from risethi) author: John Snow Labs name: distilbert_qa_risethi_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `risethi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_risethi_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772308308.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_risethi_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772308308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_risethi_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_risethi_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_risethi_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/risethi/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Court Decisions Unanimity Prediction (Portuguese) author: John Snow Labs name: legclf_court_decisions_unanimity date: 2023-04-14 tags: [pt, licensed, legal, classification, tensorflow] task: Text Classification language: pt edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Multiclass classification model which identifies the court decisions were unanimity, not_unanimity, or not_determined in the State Supreme Court. ## Predicted Entities `unanimity`, `not_unanimity`, `not_determined` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_court_decisions_unanimity_pt_1.0.0_3.0_1681479825814.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_court_decisions_unanimity_pt_1.0.0_3.0_1681479825814.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler= nlp.DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = nlp.Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") seq_classifier = legal.BertForSequenceClassification.pretrained("legclf_court_decisions_unanimity", "pt", "legal/models") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") nlpPipeline= nlp.Pipeline(stages=[ document_assembler, tokenizer, seq_classifier ]) # Example text example = spark.createDataFrame([["ACÓRDÃO/SALVO-CONDUTO PENAL E PROCESSUAL PENAL HABEAS CORPUS. CRIMES CONTRA A HONRA. ALEGAÇÃO DE INCOMPETÊNCIA DO JUÍZO, MANIPULAÇÃO DAS PROVAS NA AÇÃO ORIGINÁRIA E DEFEITO DE REPRESENTAÇÃO. PRELIMINARES AFASTADAS. IMPOSIÇÃO DE MEDIDAS CAUTELARES DIVERSAS DE PRISÃO. NECESSIDADE DE DEMONSTRAÇÃO DA ADEQUAÇÃO E PROPORCIONALIDADE DAS MEDIDAS. PRECEDENTES DO STJ. DECISÃO DESPROVIDA DE FUNDAMENTAÇÃO. MERA REFERÊNCIA À GRAVIDADE DOS FATOS NARRADOS. NULIDADE. INTELIGÊNCIA DO ARTIGO 93, IX, DA CRFB. CONSTRANGIMENTO ILEGAL EVIDENCIADO. ORDEM CONCEDIDA. 1 Observando que as penas máximas cominadas aos crimes imputados aos querelantes, em concurso material, excedem a dois anos, não há que se falar em competência dos Juizados Especiais Criminais. 2 Na estreita via do mandamus, não se faz possível avaliar eventual manipulação de provas procedida pela vítima nos autos originários. 3 Embora menos gravosas que a segregação cautelar, as medidas cautelares possuem o condão de restringir direitos individuais, razão pela qual não dispensam fundamentação acerca da sua necessidade e adequação. 4 Estando a decisão atacada totalmente desprovida de fundamentação, forçoso declarar a sua nulidade, revogando as medidas cautelares impostas."]]).toDF("text") empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) result = model.transform(example) # result is a DataFrame result.select("text", "class.result").show(truncate=200) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_6","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|375.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-6 --- layout: model title: Fast Neural Machine Translation Model from English to West Slavic languages author: John Snow Labs name: opus_mt_en_zlw date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, zlw, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `zlw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_zlw_xx_2.7.0_2.4_1609164254897.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_zlw_xx_2.7.0_2.4_1609164254897.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_zlw| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Shunichiro) author: John Snow Labs name: distilbert_qa_shunichiro_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Shunichiro`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_shunichiro_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769221544.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_shunichiro_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769221544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shunichiro_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shunichiro_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_shunichiro_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Shunichiro/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Validity Clause Binary Classifier author: John Snow Labs name: legclf_validity_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `validity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `validity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_validity_clause_en_1.0.0_3.2_1660124112127.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_validity_clause_en_1.0.0_3.2_1660124112127.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_validity_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[validity]| |[other]| |[other]| |[validity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_validity_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.98 0.97 89 validity 0.94 0.89 0.92 37 accuracy - - 0.95 126 macro-avg 0.95 0.93 0.94 126 weighted-avg 0.95 0.95 0.95 126 ``` --- layout: model title: Translate English to Finno-Ugrian languages Pipeline author: John Snow Labs name: translate_en_fiu date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, fiu, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `fiu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_fiu_xx_2.7.0_2.4_1609687072182.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_fiu_xx_2.7.0_2.4_1609687072182.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_fiu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_fiu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.fiu').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_fiu| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Mapping Entities with Corresponding ICD-10-CM Codes author: John Snow Labs name: icd10cm_mapper date: 2022-10-29 tags: [icd10cm, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Spark NLP for Healthcare 4.2.1 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps entities with their corresponding ICD-10-CM codes. ## Predicted Entities `icd10cm_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_mapper_en_4.2.1_3.0_1667082016627.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_mapper_en_4.2.1_3.0_1667082016627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel\ .pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel\ .pretrained("ner_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols("sentence", "token", "ner")\ .setOutputCol("ner_chunk") chunkerMapper = ChunkMapperModel\ .pretrained("icd10cm_mapper", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["icd10cm_code"]) mapper_pipeline = Pipeline().setStages([ document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_converter, chunkerMapper]) test_data = spark.createDataFrame([["A 35-year-old male with a history of primary leiomyosarcoma of neck, gestational diabetes mellitus diagnosed eight years prior to presentation and presented with a one-week history of polydipsia, poor appetite, and vomiting."]]).toDF("text") mapper_model = mapper_pipeline.fit(test_data) result= mapper_model.transform(test_data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel .pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols("sentence", "token", "ner") .setOutputCol("ner_chunk") val chunkerMapper = ChunkMapperModel .pretrained("icd10cm_mapper", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("mappings") .setRels(Array("icd10cm_code")) val mapper_pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_converter, chunkerMapper)) val data = Seq("A 35-year-old male with a history of primary leiomyosarcoma of neck, gestational diabetes mellitus diagnosed eight years prior to presentation and presented with a one-week history of polydipsia, poor appetite, and vomiting.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.icd10cm").predict("""A 35-year-old male with a history of primary leiomyosarcoma of neck, gestational diabetes mellitus diagnosed eight years prior to presentation and presented with a one-week history of polydipsia, poor appetite, and vomiting.""") ```

## Results ```bash +------------------------------+-------+------------+ |ner_chunk |entity |icd10cm_code| +------------------------------+-------+------------+ |primary leiomyosarcoma of neck|PROBLEM|C49.0 | |gestational diabetes mellitus |PROBLEM|O24.919 | |polydipsia |PROBLEM|R63.1 | |poor appetite |PROBLEM|R63.0 | |vomiting |PROBLEM|R11.10 | +------------------------------+-------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_mapper| |Compatibility:|Spark NLP for Healthcare 4.2.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|6.2 MB| --- layout: model title: Sentence Entity Resolver for billable ICD10-CM HCC codes (Slim) author: John Snow Labs name: sbiobertresolve_icd10cm_slim_billable_hcc date: 2021-05-21 tags: [licensed, clinical, en, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. This model has been augmented with synonyms and synonyms having low cosine similarity are dropped, making the model slim. ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for aux_label parameter in the metadata to get HCC status. The HCC status can be divided to get further information: billable status, hcc status, and hcc score. For example, in the example shared below the billable status is 1, hcc status is 1, and hcc score is 8. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.0.4_2.4_1621588560429.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.0.4_2.4_1621588560429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler().setInputCol("text").setOutputCol("document") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["document"])\ .setOutputCol("sbert_embeddings") icd10_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_icd10cm_slim_billable_hcc","en", "clinical/models")\ .setInputCols(["document", "sbert_embeddings"])\ .setOutputCol("icd10cm_code")\ .setDistanceFunction("EUCLIDEAN")\ .setReturnCosineDistances(True) bert_pipeline_icd = Pipeline(stages = [document_assembler, sbert_embedder, icd10_resolver]) data = spark.createDataFrame([["metastatic lung cancer"]]).toDF("text") results = bert_pipeline_icd.fit(data).transform(data) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("document")) .setOutputCol("sbert_embeddings") val icd10_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_icd10cm_slim_billable_hcc","en", "clinical/models") .setInputCols(Array("document", "sbert_embeddings")) .setOutputCol("icd10cm_code") .setDistanceFunction("EUCLIDEAN") .setReturnCosineDistances(True) val bert_pipeline_icd = new Pipeline().setStages(Array(document_assembler, sbert_embedder, icd10_resolver)) val data = Seq("metastatic lung cancer").toDF("text") val result = bert_pipeline_icd.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd10cm.slim_billable_hcc").predict("""metastatic lung cancer""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: Fast Neural Machine Translation Model from English to Congo Swahili author: John Snow Labs name: opus_mt_en_swc date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, swc, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `swc` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_swc_xx_2.7.0_2.4_1609167640396.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_swc_xx_2.7.0_2.4_1609167640396.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_swc| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Toxic content classifier for German author: John Snow Labs name: distilbert_base_sequence_classifier_toxicity date: 2022-02-16 tags: [toxic, distilbert, sequence_classifier, de, open_source] task: Text Classification language: de edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: DistilBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` ([link](https://huggingface.co/ml6team/distilbert-base-german-cased-toxic-comments)) and it's been trained on GermEval21 and IWG Hatespeech datasets for the German language, leveraging `Distil-BERT` embeddings and `DistilBertForSequenceClassification` for text classification purposes. ## Predicted Entities `non_toxic`, `toxic` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_toxicity_de_3.4.0_3.0_1645021339319.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_toxicity_de_3.4.0_3.0_1645021339319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['toxic'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_sequence_classifier_toxicity| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[class]| |Language:|de| |Size:|252.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - [GermEval21](https://github.com/germeval2021toxic/SharedTask/tree/main/Data%20Sets) - [IWG Hatespeech](https://github.com/UCSM-DUE/IWG_hatespeech_public) --- layout: model title: Legal Environmental Policy Document Classifier (EURLEX) author: John Snow Labs name: legclf_environmental_policy_bert date: 2023-03-06 tags: [en, legal, classification, clauses, environmental_policy, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_environmental_policy_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Environmental_Policy or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Environmental_Policy`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_environmental_policy_bert_en_1.0.0_3.0_1678111880590.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_environmental_policy_bert_en_1.0.0_3.0_1678111880590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_environmental_policy_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Environmental_Policy]| |[Other]| |[Other]| |[Environmental_Policy]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_environmental_policy_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Environmental_Policy 0.85 0.86 0.86 227 Other 0.85 0.84 0.85 216 accuracy - - 0.85 443 macro-avg 0.85 0.85 0.85 443 weighted-avg 0.85 0.85 0.85 443 ``` --- layout: model title: Sentence Entity Resolver for Clinical Abbreviations and Acronyms (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_clinical_abbreviation_acronym date: 2022-01-03 tags: [abbreviation, entity_resolver, licensed, en, clinical, acronym] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical abbreviations and acronyms to their meanings using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It is the first primitive version of abbreviation resolution and will be improved further in the following releases. ## Predicted Entities `Abbreviation Meanings` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_clinical_abbreviation_acronym_en_3.3.4_2.4_1641239524364.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_clinical_abbreviation_acronym_en_3.3.4_2.4_1641239524364.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["document", "token"])\ .setOutputCol("word_embeddings") clinical_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") \ .setInputCols(["document", "token", "word_embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal() \ .setInputCols(["document", "token", "ner"]) \ .setOutputCol("ner_chunk")\ .setWhiteList(['ABBR']) sentence_chunk_embeddings = BertSentenceChunkEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models")\ .setInputCols(["document", "ner_chunk"])\ .setOutputCol("sentence_embeddings")\ .setChunkWeight(0.5)\ .setCaseSensitive(True) abbr_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_clinical_abbreviation_acronym", "en", "clinical/models") \ .setInputCols(["sentence_embeddings"]) \ .setOutputCol("abbr_meaning")\ .setDistanceFunction("EUCLIDEAN")\ resolver_pipeline = Pipeline( stages = [ document_assembler, tokenizer, word_embeddings, clinical_ner, ner_converter, sentence_chunk_embeddings, abbr_resolver ]) text = "The patient admitted from the IR for aggressive irrigation of the Miami pouch. DISCHARGE DIAGNOSES: 1. A 58-year-old female with a history of stage 2 squamous cell carcinoma of the cervix status post total pelvic exenteration in 1991." sample_text = spark.createDataFrame([[text]]).toDF('text') abbr_result = resolver_pipeline.fit(sample_text).transform(sample_text) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("document", "token")) .setOutputCol("word_embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") .setInputCols(Array("document", "token", "word_embeddings")) .setOutputCol("ner") val ner_converter = NerConverterInternal() .setInputCols(Array("document", "token", "ner")) .setOutputCol("ner_chunk") .setWhiteList(Array("ABBR")) val sentence_chunk_embeddings = BertSentenceChunkEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols(Array("document", "ner_chunk")) .setOutputCol("sentence_embeddings") .setChunkWeight(0.5) .setCaseSensitive(True) val abbr_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_clinical_abbreviation_acronym", "en", "clinical/models") .setInputCols(Array("sentence_embeddings")) .setOutputCol("abbr_meaning") .setDistanceFunction("EUCLIDEAN") val resolver_pipeline = new Pipeline().setStages(document_assembler, tokenizer, word_embeddings, clinical_ner, ner_converter, sentence_chunk_embeddings, abbr_resolver) val sample_text = Seq("The patient admitted from the IR for aggressive irrigation of the Miami pouch. DISCHARGE DIAGNOSES: 1. A 58-year-old female with a history of stage 2 squamous cell carcinoma of the cervix status post total pelvic exenteration in 1991.").toDF("text") val abbr_result = resolver_pipeline.fit(sample_text).transform(sample_text) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.clinical_abbreviation_acronym").predict("""The patient admitted from the IR for aggressive irrigation of the Miami pouch. DISCHARGE DIAGNOSES: 1. A 58-year-old female with a history of stage 2 squamous cell carcinoma of the cervix status post total pelvic exenteration in 1991.""") ```

## Results ```bash +-------+---------+------+------------------------+-------------------------------------------------------------------------+-----------------+---------------------------------+ |sent_id|ner_chunk|entity| abbr_meaning| all_k_results|all_k_resolutions| all_k_cosine_distances| +-------+---------+------+------------------------+-------------------------------------------------------------------------+-----------------+---------------------------------+ | 0| IR| ABBR|interventional radiology|interventional radiology:::immediate-release:::(stage) IA:::intraarterial|IR:::IR:::IA:::IA|0.0156:::0.0945:::0.1046:::0.1111| +-------+---------+------+------------------------+-------------------------------------------------------------------------+-----------------+---------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_clinical_abbreviation_acronym| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[abbr_meaning]| |Language:|en| |Size:|105.3 MB| |Case sensitive:|false| --- layout: model title: Legal Chemistry Document Classifier (EURLEX) author: John Snow Labs name: legclf_chemistry_bert date: 2023-03-06 tags: [en, legal, classification, clauses, chemistry, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_chemistry_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Chemistry or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Chemistry`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_chemistry_bert_en_1.0.0_3.0_1678111646937.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_chemistry_bert_en_1.0.0_3.0_1678111646937.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_chemistry_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Chemistry]| |[Other]| |[Other]| |[Chemistry]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_chemistry_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Chemistry 0.84 0.92 0.88 180 Other 0.93 0.85 0.89 212 accuracy - - 0.88 392 macro-avg 0.88 0.89 0.88 392 weighted-avg 0.89 0.88 0.88 392 ``` --- layout: model title: Pipeline to Detect Clinical Events author: John Snow Labs name: ner_events_clinical_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_events_clinical](https://nlp.johnsnowlabs.com/2021/03/31/ner_events_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_clinical_pipeline_en_4.3.0_3.2_1678866574652.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_clinical_pipeline_en_4.3.0_3.2_1678866574652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_events_clinical_pipeline", "en", "clinical/models") text = '''The patient presented to the emergency room last evening.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_events_clinical_pipeline", "en", "clinical/models") val text = "The patient presented to the emergency room last evening." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.events_clinical.pipeline").predict("""The patient presented to the emergency room last evening.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------------|--------:|------:|:--------------|-------------:| | 0 | presented | 12 | 20 | OCCURRENCE | 0.7132 | | 1 | the emergency room | 25 | 42 | CLINICAL_DEPT | 0.723267 | | 2 | last evening | 44 | 55 | DATE | 0.90555 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English LongformerForQuestionAnswering model (from abhijithneilabraham) author: John Snow Labs name: longformer_qa_covid date: 2022-06-26 tags: [en, open_source, longformer, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `longformer_covid_qa` is a English model originally trained by `abhijithneilabraham`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_qa_covid_en_4.0.0_3.0_1656255806950.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_qa_covid_en_4.0.0_3.0_1656255806950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_qa_covid","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_qa_covid","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.covid.longformer").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_qa_covid| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|556.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/abhijithneilabraham/longformer_covid_qa - https://github.com/abhijithneilabraham/Covid-QA --- layout: model title: Smaller BERT Embeddings (L-8_H-768_A-12) author: John Snow Labs name: small_bert_L8_768 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L8_768_en_2.6.0_2.4_1598345245072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L8_768_en_2.6.0_2.4_1598345245072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L8_768", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L8_768", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L8_768').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L8_768_embeddings I [-0.17749905586242676, 0.12339337915182114, -0... love [-0.19556283950805664, -0.24686720967292786, 0... NLP [-0.5068103671073914, -0.5276650786399841, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L8_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ftorres) author: John Snow Labs name: distilbert_qa_ftorres_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `ftorres`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ftorres_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770750585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ftorres_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770750585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ftorres_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ftorres_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ftorres_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ftorres/distilbert-base-uncased-finetuned-squad --- layout: model title: Match Pattern author: John Snow Labs name: match_pattern date: 2022-06-19 tags: [en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The match_pattern is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and matches pattrens . It performs most of the common text processing tasks on your dataframe ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/match_pattern_en_4.0.0_3.0_1655653735848.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/match_pattern_en_4.0.0_3.0_1655653735848.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("match_pattern", "en", "clinical/models") result = pipeline.annotate("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|match_pattern| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|29.0 KB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - RegexMatcherModel --- layout: model title: Word2Vec Embeddings in Emiliano-Romagnolo (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, eml, open_source] task: Embeddings language: eml edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eml_3.4.1_3.0_1647294563985.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eml_3.4.1_3.0_1647294563985.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ceb", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ceb", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ceb').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ceb| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from huggingface-course) author: John Snow Labs name: bert_qa_huggingface_course_bert_finetuned_squad_accelerate date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model orginally trained by `huggingface-course`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_huggingface_course_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535856581.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_huggingface_course_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535856581.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_huggingface_course_bert_finetuned_squad_accelerate","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_huggingface_course_bert_finetuned_squad_accelerate","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.accelerate.by_huggingface-course").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_huggingface_course_bert_finetuned_squad_accelerate| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huggingface-course/bert-finetuned-squad-accelerate --- layout: model title: Legal Underwriting Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_underwriting_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, underwriting, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_underwriting_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `underwriting-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `underwriting-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_underwriting_agreement_bert_en_1.0.0_3.0_1670349963889.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_underwriting_agreement_bert_en_1.0.0_3.0_1670349963889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_underwriting_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[underwriting-agreement]| |[other]| |[other]| |[underwriting-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_underwriting_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.97 0.97 65 underwriting-agreement 0.94 0.94 0.94 36 accuracy - - 0.96 101 macro-avg 0.96 0.96 0.96 101 weighted-avg 0.96 0.96 0.96 101 ``` --- layout: model title: Translate Walloon to English Pipeline author: John Snow Labs name: translate_wa_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, wa, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `wa` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_wa_en_xx_2.7.0_2.4_1609690770769.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_wa_en_xx_2.7.0_2.4_1609690770769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_wa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_wa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.wa.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_wa_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Assertion Status from Oncology Treatments author: John Snow Labs name: assertion_oncology_treatment_binary_wip date: 2022-10-11 tags: [licensed, clinical, oncology, en, assertion, treatment] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of oncology treatment entities. The model identifies if the treatment has been used (Present_Or_Past status), or if it is mentioned but in hypothetical or absent sentences (Hypothetical_Or_Absent status). ## Predicted Entities `Hypothetical_Or_Absent`, `Present_Or_Past` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_treatment_binary_wip_en_4.0.0_3.0_1665521859244.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_treatment_binary_wip_en_4.0.0_3.0_1665521859244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:-------------|:---------------|:-----------------------| | mastectomy | Cancer_Surgery | Present_Or_Past | | chemotherapy | Chemotherapy | Hypothetical_Or_Absent | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_treatment_binary_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Hypothetical_Or_Absent 0.78 0.83 0.81 170.0 Present_Or_Past 0.81 0.76 0.78 160.0 macro-avg 0.80 0.79 0.79 330.0 weighted-avg 0.79 0.79 0.79 330.0 ``` --- layout: model title: Sundanese asr_wav2vec2_indonesian_javanese_sundanese TFWav2Vec2ForCTC from indonesian-nlp author: John Snow Labs name: pipeline_asr_wav2vec2_indonesian_javanese_sundanese date: 2022-09-24 tags: [wav2vec2, su, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: su edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_indonesian_javanese_sundanese` is a Sundanese model originally trained by indonesian-nlp. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_indonesian_javanese_sundanese_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_indonesian_javanese_sundanese_su_4.2.0_3.0_1664038727353.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_indonesian_javanese_sundanese_su_4.2.0_3.0_1664038727353.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_indonesian_javanese_sundanese', lang = 'su') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_indonesian_javanese_sundanese", lang = "su") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_indonesian_javanese_sundanese| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|su| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Spanish RobertaForQuestionAnswering (from BSC-TeMU) author: John Snow Labs name: roberta_qa_BSC_TeMU_roberta_base_bne_sqac date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-sqac` is a Spanish model originally trained by `BSC-TeMU`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_BSC_TeMU_roberta_base_bne_sqac_es_4.0.0_3.0_1655730098181.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_BSC_TeMU_roberta_base_bne_sqac_es_4.0.0_3.0_1655730098181.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_BSC_TeMU_roberta_base_bne_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_BSC_TeMU_roberta_base_bne_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.sqac.roberta.base.by_BSC-TeMU").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_BSC_TeMU_roberta_base_bne_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/BSC-TeMU/roberta-base-bne-sqac - https://github.com/PlanTL-SANIDAD/lm-spanish - http://www.bne.es/en/Inicio/index.html - https://arxiv.org/abs/1907.11692 - https://arxiv.org/abs/2107.07253 --- layout: model title: Translate Eastern Malayo-Polynesian languages to English Pipeline author: John Snow Labs name: translate_pqe_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, pqe, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `pqe` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_pqe_en_xx_2.7.0_2.4_1609699605019.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_pqe_en_xx_2.7.0_2.4_1609699605019.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_pqe_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_pqe_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.pqe.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_pqe_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Persian XLMRobertaForTokenClassification Base Cased model (from BK-V) author: John Snow Labs name: xlmroberta_ner_base_finetuned_peyma date: 2022-08-14 tags: [fa, open_source, xlm_roberta, ner] task: Named Entity Recognition language: fa edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-peyma-fa` is a Persian model originally trained by `BK-V`. ## Predicted Entities `LOC`, `PER`, `MON`, `ORG`, `TIM`, `PCT`, `DAT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_peyma_fa_4.1.0_3.0_1660447374860.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_peyma_fa_4.1.0_3.0_1660447374860.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_peyma","fa") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_peyma","fa") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_peyma| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|843.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/BK-V/xlm-roberta-base-finetuned-peyma-fa --- layout: model title: Estonian CamemBert Embeddings (from EMBEDDIA) author: John Snow Labs name: camembert_embeddings_est_roberta date: 2022-05-31 tags: [et, open_source, camembert, embeddings] task: Embeddings language: et edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `est-roberta` is a Estonian model orginally trained by `EMBEDDIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_est_roberta_et_3.4.4_3.0_1653991675972.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_est_roberta_et_3.4.4_3.0_1653991675972.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_est_roberta| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|et| |Size:|280.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/EMBEDDIA/est-roberta - https://camembert-model.fr/ --- layout: model title: Vietnamese T5ForConditionalGeneration Base Cased model (from VietAI) author: John Snow Labs name: t5_vit5_base date: 2023-01-31 tags: [vi, open_source, t5, tensorflow] task: Text Generation language: vi edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `vit5-base` is a Vietnamese model originally trained by `VietAI`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_vit5_base_vi_4.3.0_3.0_1675158262308.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_vit5_base_vi_4.3.0_3.0_1675158262308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_vit5_base","vi") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_vit5_base","vi") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_vit5_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|vi| |Size:|485.0 MB| ## References - https://huggingface.co/VietAI/vit5-base - https://github.com/vietai/ViT5 --- layout: model title: Extract relations between problem, treatment and test entities (ReDL) author: John Snow Labs name: redl_clinical_biobert date: 2023-01-06 tags: [licensed, clinical, en, relation_extraction, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations like `TrIP` : a certain treatment has improved a medical problem and 7 other such relations between problem, treatment and test entities. ## Predicted Entities `PIP`, `TeCP`, `TeRP`, `TrAP`, `TrCP`, `TrIP`, `TrNAP`, `TrWP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_clinical_biobert_en_4.2.4_3.0_1673020165617.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_clinical_biobert_en_4.2.4_3.0_1673020165617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = sparknlp.annotators.Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") words_embedder = WordEmbeddingsModel() \ .pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"]) \ .setOutputCol("embeddings") ner_tagger = MedicalNerModel() \ .pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens", "embeddings"]) \ .setOutputCol("ner_tags") ner_converter = NerConverter() \ .setInputCols(["sentences", "tokens", "ner_tags"]) \ .setOutputCol("ner_chunks") dependency_parser = DependencyParserModel() \ .pretrained("dependency_conllu", "en") \ .setInputCols(["sentences", "pos_tags", "tokens"]) \ .setOutputCol("dependencies") # Set a filter on pairs of named entities which will be treated as relation candidates re_ner_chunk_filter = RENerChunksFilter() \ .setInputCols(["ner_chunks", "dependencies"])\ .setMaxSyntacticDistance(10)\ .setOutputCol("re_ner_chunks")\ .setRelationPairs(["problem-test", "problem-treatment"]) # The dataset this model is trained to is sentence-wise. # This model can also be trained on document-level relations - in which case, while predicting, use "document" instead of "sentence" as input. re_model = RelationExtractionDLModel()\ .pretrained('redl_clinical_biobert', 'en', "clinical/models") \ .setPredictionThreshold(0.5)\ .setInputCols(["re_ner_chunks", "sentences"]) \ .setOutputCol("relations") pipeline = Pipeline(stages=[documenter, sentencer, tokenizer, pos_tagger, words_embedder, ner_tagger, ner_converter, dependency_parser, re_ner_chunk_filter, re_model]) text ="""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation. Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge . """ data = spark.createDataFrame([[text]]).toDF("text") p_model = pipeline.fit(data) result = p_model.transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentencer = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols(Array("sentences")) .setOutputCol("tokens") val pos_tagger = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val ner_tagger = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") // Set a filter on pairs of named entities which will be treated as relation candidates val re_ner_chunk_filter = RENerChunksFilter() .setInputCols(Array("ner_chunks", "dependencies")) .setMaxSyntacticDistance(10) .setOutputCol("re_ner_chunks") .setRelationPairs(Array("problem-test", "problem-treatment")) // The dataset this model is trained to is sentence-wise. // This model can also be trained on document-level relations - in which case, while predicting, use "document" instead of "sentence" as input. val re_model = RelationExtractionDLModel() .pretrained("redl_clinical_biobert", "en", "clinical/models") .setPredictionThreshold(0.5) .setInputCols(Array("re_ner_chunks", "sentences")) .setOutputCol("relations") val pipeline = new Pipeline().setStages(Array(documenter, sentencer, tokenizer, pos_tagger, words_embedder, ner_tagger, ner_converter, dependency_parser, re_ner_chunk_filter, re_model)) val data = Seq("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation. Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.clinical").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation. Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge . """) ```

## Results ```bash +--------+---------+-------------+-----------+--------------------+---------+-------------+-----------+--------------------+----------+ |relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +--------+---------+-------------+-----------+--------------------+---------+-------------+-----------+--------------------+----------+ | TrAP|TREATMENT| 512| 522| amoxicillin| PROBLEM| 528| 556|a respiratory tra...|0.99796957| | TrAP|TREATMENT| 571| 579| metformin| PROBLEM| 617| 620| T2DM|0.99757993| | TrAP|TREATMENT| 599| 611| dapagliflozin| PROBLEM| 659| 661| HTG| 0.996036| | TrAP| PROBLEM| 617| 620| T2DM|TREATMENT| 626| 637| atorvastatin| 0.9693424| | TrAP| PROBLEM| 617| 620| T2DM|TREATMENT| 643| 653| gemfibrozil|0.99460286| | TeRP| TEST| 739| 758|Physical examination| PROBLEM| 796| 810| dry oral mucosa|0.99775106| | TeRP| TEST| 830| 854|her abdominal exa...| PROBLEM| 875| 884| tenderness|0.99272937| | TeRP| TEST| 830| 854|her abdominal exa...| PROBLEM| 888| 895| guarding| 0.9840321| | TeRP| TEST| 830| 854|her abdominal exa...| PROBLEM| 902| 909| rigidity| 0.9883966| | TeRP| TEST| 1246| 1258| blood samples| PROBLEM| 1265| 1274| hemolyzing| 0.9534202| | TeRP| TEST| 1507| 1517| her glucose| PROBLEM| 1553| 1566| still elevated| 0.9464761| | TeRP| PROBLEM| 1553| 1566| still elevated| TEST| 1576| 1592| serum bicarbonate| 0.9428323| | TeRP| PROBLEM| 1553| 1566| still elevated| TEST| 1656| 1661| lipase| 0.9558198| | TeRP| PROBLEM| 1553| 1566| still elevated| TEST| 1670| 1672| U/L| 0.9214444| | TeRP| TEST| 1676| 1702|The β-hydroxybuty...| PROBLEM| 1733| 1740| elevated| 0.9863963| | TrAP|TREATMENT| 1937| 1951| an insulin drip| PROBLEM| 1957| 1961| euDKA| 0.9852455| | O| PROBLEM| 1957| 1961| euDKA| TEST| 1991| 2003| the anion gap|0.94141793| | O| PROBLEM| 1957| 1961| euDKA| TEST| 2015| 2027| triglycerides| 0.9622529| +--------+---------+-------------+-----------+--------------------+---------+-------------+-----------+--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_clinical_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|402.0 MB| ## Benchmarking ```bash Relation Recall Precision F1 Support PIP 0.859 0.878 0.869 1435 TeCP 0.629 0.782 0.697 337 TeRP 0.903 0.929 0.916 2034 TrAP 0.872 0.866 0.869 1693 TrCP 0.641 0.677 0.659 340 TrIP 0.517 0.796 0.627 151 TrNAP 0.402 0.672 0.503 112 TrWP 0.257 0.824 0.392 109 Avg. 0.635 0.803 0.691 - ``` --- layout: model title: English RobertaForQuestionAnswering (from comacrae) author: John Snow Labs name: roberta_qa_roberta_unaugv3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-unaugv3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_unaugv3_en_4.0.0_3.0_1655738558018.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_unaugv3_en_4.0.0_3.0_1655738558018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_unaugv3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_unaugv3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.unaugv3.by_comacrae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_unaugv3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/comacrae/roberta-unaugv3 --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_specter_model_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `specter-bert-model_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_specter_model_squad2.0_en_4.0.0_3.0_1657192817801.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_specter_model_squad2.0_en_4.0.0_3.0_1657192817801.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_specter_model_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_specter_model_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_specter_model_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/specter-bert-model_squad2.0 --- layout: model title: English BertForQuestionAnswering Cased model (from SebastianS) author: John Snow Labs name: bert_qa_sebastians_finetuned_squad_accelera date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model originally trained by `SebastianS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sebastians_finetuned_squad_accelera_en_4.0.0_3.0_1657187023304.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sebastians_finetuned_squad_accelera_en_4.0.0_3.0_1657187023304.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sebastians_finetuned_squad_accelera","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_sebastians_finetuned_squad_accelera","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sebastians_finetuned_squad_accelera| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SebastianS/bert-finetuned-squad-accelerate --- layout: model title: Financial Sentiment Analysis author: John Snow Labs name: finclf_bert_sentiment date: 2022-09-06 tags: [en, finance, sentiment, analysis, classification, licensed] task: Sentiment Analysis language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Sentiment Analysis fine-tuned model on 12K+ manually annotated (positive, negative, neutral) analyst reports on top of Financial Bert Embeddings. This model achieves superior performance on financial tone analysis task. ## Predicted Entities `positive`, `negative`, `neutral` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_FINANCE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_bert_sentiment_en_1.0.0_3.2_1662469460654.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_bert_sentiment_en_1.0.0_3.2_1662469460654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+----------+ | text| result| +--------------------+----------+ |Stocks rallied an...|[Positive]| +--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_bert_sentiment| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|412.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotations on financial reports ## Benchmarking ```bash label precision recall f1-score support neutral 0.91 0.87 0.89 588 positive 0.76 0.81 0.78 251 negative 0.83 0.87 0.85 131 accuracy - - 0.86 970 macro-avg 0.83 0.85 0.84 970 weighted-avg 0.86 0.86 0.86 970 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from saburbutt) author: John Snow Labs name: xlm_roberta_qa_xlmroberta_large_tweetqa date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlmroberta_large_tweetqa` is a English model originally trained by `saburbutt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmroberta_large_tweetqa_en_4.0.0_3.0_1655997777896.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlmroberta_large_tweetqa_en_4.0.0_3.0_1655997777896.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlmroberta_large_tweetqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlmroberta_large_tweetqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.trivia.xlmr_roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlmroberta_large_tweetqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saburbutt/xlmroberta_large_tweetqa --- layout: model title: English BertForTokenClassification Base Uncased model (from ml6team) author: John Snow Labs name: bert_token_classifier_base_uncased_city_country_ner date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-city-country-ner` is a English model originally trained by `ml6team`. ## Predicted Entities `CITY`, `COUNTRY` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_uncased_city_country_ner_en_4.2.4_3.0_1669815124980.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_uncased_city_country_ner_en_4.2.4_3.0_1669815124980.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_uncased_city_country_ner","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_uncased_city_country_ner","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_base_uncased_city_country_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|256| ## References - https://huggingface.co/ml6team/bert-base-uncased-city-country-ner - https://www.cs.utexas.edu/~eunsol/html_pages/open_entity.html --- layout: model title: German asr_wav2vec2_xls_r_300m_english_by_aware_ai TFWav2Vec2ForCTC from aware-ai author: John Snow Labs name: asr_wav2vec2_xls_r_300m_english_by_aware_ai date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_english_by_aware_ai` is a German model originally trained by aware-ai. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_english_by_aware_ai_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_english_by_aware_ai_de_4.2.0_3.0_1664097408175.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_english_by_aware_ai_de_4.2.0_3.0_1664097408175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_english_by_aware_ai", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_english_by_aware_ai", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_english_by_aware_ai| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Korean Bert Embeddings Cased model (from onlydj96) author: John Snow Labs name: bert_embeddings_pretrain date: 2023-02-20 tags: [open_source, bert, bert_embeddings, bertformaskedlm, ko, tensorflow] task: Embeddings language: ko edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_pretrain` is a Korean model originally trained by `onlydj96`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_pretrain_ko_4.3.0_3.0_1676925661631.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_pretrain_ko_4.3.0_3.0_1676925661631.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_pretrain","ko") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_pretrain","ko") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_pretrain| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ko| |Size:|415.5 MB| |Case sensitive:|true| ## References https://huggingface.co/onlydj96/bert_pretrain --- layout: model title: Pipeline to Detect Temporal Relations for Clinical Events (Enriched) author: John Snow Labs name: re_temporal_events_enriched_clinical_pipeline date: 2023-06-13 tags: [licensed, clinical, relation_extraction, event, enriched, en] task: Relation Extraction language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_temporal_events_enriched_clinical](https://nlp.johnsnowlabs.com/2020/09/28/re_temporal_events_enriched_clinical_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_temporal_events_enriched_clinical_pipeline_en_4.4.4_3.2_1686664970911.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_temporal_events_enriched_clinical_pipeline_en_4.4.4_3.2_1686664970911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("re_temporal_events_enriched_clinical_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 56-year-old right-handed female with longstanding intermittent right low back pain, who was involved in a motor vehicle accident in September of 2005. At that time, she did not notice any specific injury, but five days later, she started getting abnormal right low back pain.") ``` ```scala val pipeline = new PretrainedPipeline("re_temporal_events_enriched_clinical_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 56-year-old right-handed female with longstanding intermittent right low back pain, who was involved in a motor vehicle accident in September of 2005. At that time, she did not notice any specific injury, but five days later, she started getting abnormal right low back pain.") ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.temproal_enriched.pipeline").predict("""The patient is a 56-year-old right-handed female with longstanding intermittent right low back pain, who was involved in a motor vehicle accident in September of 2005. At that time, she did not notice any specific injury, but five days later, she started getting abnormal right low back pain.""") ```

## Results ```bash Results +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | +====+============+===========+=================+===============+===============================================+============+=================+===============+==========================+==============+ | 0 | OVERLAP | PROBLEM | 54 | 98 | longstanding intermittent right low back pain | OCCURRENCE | 121 | 144 | a motor vehicle accident | 0.532308 | +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ | 1 | AFTER | DATE | 171 | 179 | that time | PROBLEM | 201 | 219 | any specific injury | 0.577288 | +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_temporal_events_enriched_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: Dutch BertForMaskedLM Base Cased model (from GroNLP) author: John Snow Labs name: bert_embeddings_base_dutch_cased date: 2022-12-02 tags: [nl, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: nl edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-dutch-cased` is a Dutch model originally trained by `GroNLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_dutch_cased_nl_4.2.4_3.0_1670016541889.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_dutch_cased_nl_4.2.4_3.0_1670016541889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_dutch_cased","nl") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_dutch_cased","nl") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_dutch_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|nl| |Size:|409.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/GroNLP/bert-base-dutch-cased - https://www.semanticscholar.org/author/Wietse-de-Vries/144611157 - https://www.semanticscholar.org/author/Andreas-van-Cranenburgh/2791585 - https://www.semanticscholar.org/author/Arianna-Bisazza/3242253 - https://www.semanticscholar.org/author/Tommaso-Caselli/1864635 - https://www.semanticscholar.org/author/Gertjan-van-Noord/143715131 - https://www.semanticscholar.org/author/M.-Nissim/2742475 - https://arxiv.org/abs/1912.09582 - https://github.com/wietsedv/bertje - https://www.semanticscholar.org/paper/BERTje%3A-A-Dutch-BERT-Model-Vries-Cranenburgh/a4d5e425cac0bf84c86c0c9f720b6339d6288ffa - https://www.clips.uantwerpen.be/conll2002/ner/ - https://ivdnt.org/downloads/taalmaterialen/tstc-sonar-corpus - https://github.com/google-research/bert/blob/master/multilingual.md - http://textdata.nl - https://github.com/iPieter/RobBERT - https://universaldependencies.org/treebanks/nl_lassysmall/index.html - https://github.com/google-research/bert/blob/master/multilingual.md - http://textdata.nl - https://github.com/iPieter/RobBERT --- layout: model title: Chinese BertForQuestionAnswering model (from jackh1995) author: John Snow Labs name: bert_qa_roberta_base_chinese_extractive_qa_scratch date: 2022-06-02 tags: [zh, open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-chinese-extractive-qa-scratch` is a Chinese model orginally trained by `jackh1995`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_qa_scratch_zh_4.0.0_3.0_1654189279202.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_qa_scratch_zh_4.0.0_3.0_1654189279202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_roberta_base_chinese_extractive_qa_scratch","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_roberta_base_chinese_extractive_qa_scratch","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.base.by_jackh1995").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_roberta_base_chinese_extractive_qa_scratch| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|408.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jackh1995/roberta-base-chinese-extractive-qa-scratch --- layout: model title: Dutch BERT Base Cased Embedding author: John Snow Labs name: bert_base_cased date: 2021-09-07 tags: [dutch, open_source, bert_embeddings, cased, nl] task: Embeddings language: nl edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BERTje is a Dutch pre-trained BERT model developed at the University of Groningen. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_nl_3.2.2_3.0_1630999717658.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_nl_3.2.2_3.0_1630999717658.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_cased", "nl") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_cased", "nl") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("nl.embed.bert.base_cased").predict("""Put your text here.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("w2v_cc_300d","de","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("w2v_cc_300d","de","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("de.embed.w2v").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``w2v_cc_300d``. {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | w2v_cc_300d | | Type: | WordEmbeddingsModel | | Compatibility: | Healthcare NLP 2.5.5+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | de | | Dimension: | 300.0 | {:.h2_title} ## Data Source FastText common crawl word embeddings for Germany https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: Sentence Detection in Ukrainian Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [open_source, uk, sentence_detection] task: Sentence Detection language: uk edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_uk_3.2.0_3.0_1630322414306.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_uk_3.2.0_3.0_1630322414306.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "uk") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""Шукаєте чудове джерело англійського читання абзаців? Ви потрапили в потрібне місце. Згідно з останнім дослідженням, звичка читати у сучасної молоді стрімко знижується. Вони не можуть зосередитися на даному абзаці читання англійською мовою більше кількох секунд! Крім того, читання було і є невід’ємною частиною всіх конкурсних іспитів. Отже, як покращити свої навички читання? Відповідь на це питання насправді інше питання: Яка користь від навичок читання? Основна мета читання - «мати сенс».""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "uk") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("Шукаєте чудове джерело англійського читання абзаців? Ви потрапили в потрібне місце. Згідно з останнім дослідженням, звичка читати у сучасної молоді стрімко знижується. Вони не можуть зосередитися на даному абзаці читання англійською мовою більше кількох секунд! Крім того, читання було і є невід’ємною частиною всіх конкурсних іспитів. Отже, як покращити свої навички читання? Відповідь на це питання насправді інше питання: Яка користь від навичок читання? Основна мета читання - «мати сенс».").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('uk.sentence_detector').predict('Шукаєте чудове джерело англійського читання абзаців? Ви потрапили в потрібне місце. Згідно з останнім дослідженням, звичка читати у сучасної молоді стрімко знижується. Вони не можуть зосередитися на даному абзаці читання англійською мовою більше кількох секунд! Крім того, читання було і є невід’ємною частиною всіх конкурсних іспитів. Отже, як покращити свої навички читання? Відповідь на це питання насправді інше питання: Яка користь від навичок читання? Основна мета читання - «мати сенс».', output_level ='sentence') ```

## Results ```bash +-----------------------------------------------------------------------------------------------+ |result | +-----------------------------------------------------------------------------------------------+ |[Шукаєте чудове джерело англійського читання абзаців?] | |[Ви потрапили в потрібне місце.] | |[Згідно з останнім дослідженням, звичка читати у сучасної молоді стрімко знижується.] | |[Вони не можуть зосередитися на даному абзаці читання англійською мовою більше кількох секунд!]| |[Крім того, читання було і є невід’ємною частиною всіх конкурсних іспитів.] | |[Отже, як покращити свої навички читання?] | |[Відповідь на це питання насправді інше питання:] | |[Яка користь від навичок читання?] | |[Основна мета читання - «мати сенс».] | +-----------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|uk| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Pipeline to Detect Drug Chemicals author: John Snow Labs name: bert_token_classifier_ner_drugs_pipeline date: 2022-03-21 tags: [licensed, ner, drugs, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_drugs](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_drugs_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_drugs_pipeline_en_3.4.1_3.0_1647864317028.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_drugs_pipeline_en_3.4.1_3.0_1647864317028.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_ner_drugs_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_ner_drugs_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_ade.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.""") ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |potassium |DrugChem | |nucleotide |DrugChem | |anthracyclines|DrugChem | |taxanes |DrugChem | |vinorelbine |DrugChem | |vinorelbine |DrugChem | |anthracyclines|DrugChem | |taxanes |DrugChem | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_drugs_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverter --- layout: model title: English RobertaForQuestionAnswering (from Palak) author: John Snow Labs name: roberta_qa_distilroberta_base_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-base_squad` is a English model originally trained by `Palak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_base_squad_en_4.0.0_3.0_1655728407851.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_base_squad_en_4.0.0_3.0_1655728407851.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilroberta_base_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_distilroberta_base_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.distilled_base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilroberta_base_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Palak/distilroberta-base_squad --- layout: model title: German asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527 date: 2022-09-26 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527_de_4.2.0_3.0_1664190631948.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527_de_4.2.0_3.0_1664190631948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_vp_100k_accent_germany_10_austria_0_s527| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Arabic BertForMaskedLM Base Cased model (from CAMeL-Lab) author: John Snow Labs name: bert_embeddings_base_arabic_camel_msa_quarter date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabic-camelbert-msa-quarter` is a Arabic model originally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_quarter_ar_4.2.4_3.0_1670016162499.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabic_camel_msa_quarter_ar_4.2.4_3.0_1670016162499.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa_quarter","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabic_camel_msa_quarter","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabic_camel_msa_quarter| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-quarter - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: Fast Neural Machine Translation Model from English to Igbo author: John Snow Labs name: opus_mt_en_ig date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ig, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ig` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ig_xx_2.7.0_2.4_1609169417196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ig_xx_2.7.0_2.4_1609169417196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ig| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Javanese RobertaForMaskedLM Small Cased model (from w11wo) author: John Snow Labs name: roberta_embeddings_javanese_small date: 2022-12-12 tags: [jv, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: jv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `javanese-roberta-small` is a Javanese model originally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_javanese_small_jv_4.2.4_3.0_1670858755321.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_javanese_small_jv_4.2.4_3.0_1670858755321.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_javanese_small","jv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_javanese_small","jv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_javanese_small| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|jv| |Size:|468.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-roberta-small - https://arxiv.org/abs/1907.11692 - https://github.com/sgugger - https://github.com/piegu/fastai-projects/blob/master/finetuning-English-GPT2-any-language-Portuguese-HuggingFace-fastaiv2.ipynb - https://w11wo.github.io/ --- layout: model title: Detect PHI for Deidentification in Romanian (Word2Vec) author: John Snow Labs name: ner_deid_subentity date: 2022-06-27 tags: [ner, deidentification, word2vec, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CITY`, `COUNTRY`, `DATE`, `DOCTOR`, `EMAIL`, `FAX`, `HOSPITAL`, `IDNUM`, `LOCATION-OTHER`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `ZIP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ro_4.0.0_3.0_1656316441636.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ro_4.0.0_3.0_1656316441636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|HOSPITAL | |Drumul Oprea Nr. |STREET | |Vaslui |CITY | |737405 |ZIP | |+40(235)413773 |PHONE | |25 May 2022 |DATE | |BUREAN MARIA |PATIENT | |77 |AGE | |Agota Evelyn |DOCTOR | |2450502264401 |IDNUM | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|15.1 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.96 0.99 0.97 1235 CITY 0.97 0.95 0.96 307 COUNTRY 0.92 0.74 0.82 115 DATE 0.94 0.89 0.91 5006 DOCTOR 0.96 0.96 0.96 2064 EMAIL 1.00 1.00 1.00 8 FAX 1.00 0.95 0.97 56 HOSPITAL 0.78 0.83 0.80 919 IDNUM 0.98 1.00 0.99 239 LOCATION-OTHER 1.00 0.85 0.92 13 MEDICALRECORD 1.00 1.00 1.00 455 ORGANIZATION 0.34 0.41 0.37 82 PATIENT 0.85 0.90 0.87 954 PHONE 0.97 0.98 0.98 315 PROFESSION 0.87 0.80 0.83 173 STREET 0.99 0.99 0.99 174 ZIP 0.99 0.97 0.98 140 micro-avg 0.92 0.91 0.92 12255 macro-avg 0.91 0.89 0.90 12255 weighted-avg 0.93 0.91 0.92 12255 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Chem_Modified_BioBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Chem-Modified-BioBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_BioBERT_384_en_4.0.0_3.0_1657108284010.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Chem_Modified_BioBERT_384_en_4.0.0_3.0_1657108284010.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Chem_Modified_BioBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Chem-Modified-BioBERT-384 --- layout: model title: English Named Entity Recognition (Large, ConLL) author: John Snow Labs name: roberta_ner_roberta_large_ner_english date: 2022-05-03 tags: [roberta, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-large-ner-english` is a English model orginally trained by `Jean-Baptiste`. ## Predicted Entities `MISC`, `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_large_ner_english_en_3.4.2_3.0_1651593899023.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_large_ner_english_en_3.4.2_3.0_1651593899023.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_roberta_large_ner_english| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Jean-Baptiste/roberta-large-ner-english - https://medium.com/@jean-baptiste.polle/lstm-model-for-email-signature-detection-8e990384fefa --- layout: model title: Financial Acquisitions Section Binary Classifier author: John Snow Labs name: finclf_acquisitions_item date: 2022-11-03 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `acquisitions` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Finance Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `acquisitions` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_acquisitions_item_en_1.0.0_3.0_1667484190818.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_acquisitions_item_en_1.0.0_3.0_1667484190818.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_acquisitions_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[acquisitions]| |[other]| |[other]| |[acquisitions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_acquisitions_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support acquisitions 0.95 0.89 0.92 136 other 0.96 0.98 0.97 412 accuracy - - 0.96 548 macro-avg 0.95 0.94 0.95 548 weighted-avg 0.96 0.96 0.96 548 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from cometrain) author: John Snow Labs name: t5_stocks_news date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `stocks-news-t5` is a English model originally trained by `cometrain`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_stocks_news_en_4.3.0_3.0_1675107545220.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_stocks_news_en_4.3.0_3.0_1675107545220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_stocks_news","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_stocks_news","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_stocks_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|277.2 MB| ## References - https://huggingface.co/cometrain/stocks-news-t5 - https://www.kaggle.com/datasets/sbhatti/financial-sentiment-analysis --- layout: model title: Spanish BERT Base Uncased Embedding author: John Snow Labs name: bert_base_uncased date: 2021-09-07 tags: [uncased, spanish, bert_embeddings, open_source, es] task: Embeddings language: es edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BETO is a BERT model trained on a big Spanish corpus. BETO is of size similar to a BERT-Base and was trained with the Whole Word Masking technique. Below you find Tensorflow and Pytorch checkpoints for the uncased and cased versions, as well as some results for Spanish benchmarks comparing BETO with Multilingual BERT as well as other (not BERT-based) models. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_uncased_es_3.2.2_3.0_1630999639144.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_uncased_es_3.2.2_3.0_1630999639144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_uncased", "es") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_uncased", "es") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.bert.base_uncased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_uncased| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Case sensitive:|true| ## Data Source The model is imported from: https://huggingface.co/dccuchile/bert-base-spanish-wwm-uncased --- layout: model title: Legal Indemnity Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_indemnity_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, indemnity, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_indemnity_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `indemnity-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `indemnity-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_agreement_bert_en_1.0.0_3.0_1670349475264.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_agreement_bert_en_1.0.0_3.0_1670349475264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_indemnity_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[indemnity-agreement]| |[other]| |[other]| |[indemnity-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnity_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support indemnity-agreement 0.88 0.82 0.85 28 other 0.86 0.91 0.89 35 accuracy - - 0.87 63 macro-avg 0.87 0.87 0.87 63 weighted-avg 0.87 0.87 0.87 63 ``` --- layout: model title: English DistilBertForQuestionAnswering Cased model (from saraks) author: John Snow Labs name: distilbert_qa_cuad_agreement_date_08_25 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cuad-distil-agreement_date-08-25` is a English model originally trained by `saraks`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_agreement_date_08_25_en_4.3.0_3.0_1672765994991.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_agreement_date_08_25_en_4.3.0_3.0_1672765994991.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_agreement_date_08_25","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_agreement_date_08_25","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cuad_agreement_date_08_25| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saraks/cuad-distil-agreement_date-08-25 --- layout: model title: Legal Restricted Stock Unit Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_restricted_stock_unit_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, restricted, stock, unit, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_restricted_stock_unit_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `restricted-stock-unit-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `restricted-stock-unit-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_restricted_stock_unit_agreement_bert_en_1.0.0_3.0_1670349793942.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_restricted_stock_unit_agreement_bert_en_1.0.0_3.0_1670349793942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_restricted_stock_unit_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[restricted-stock-unit-agreement]| |[other]| |[other]| |[restricted-stock-unit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_restricted_stock_unit_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.94 0.94 35 restricted-stock-unit-agreement 0.95 0.95 0.95 42 accuracy - - 0.95 77 macro-avg 0.95 0.95 0.95 77 weighted-avg 0.95 0.95 0.95 77 ``` --- layout: model title: Polish Lemmatizer author: John Snow Labs name: lemma date: 2020-05-03 18:18:00 +0800 task: Lemmatization language: pl edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, pl] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_pl_2.5.0_2.4_1588518491035.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_pl_2.5.0_2.4_1588518491035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "pl") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "pl") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Oprócz bycia królem północy, John Snow jest angielskim lekarzem i liderem w rozwoju anestezjologii i higieny medycznej."""] lemma_df = nlu.load('pl.lemma').predict(text, output_level='token') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=5, result='oprócz', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=7, end=11, result='bycia', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=13, end=18, result='król', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=20, end=26, result='północ', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=27, end=27, result=',', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|pl| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Law Spanish Text Classification (from `hackathon-pln-es`) author: John Snow Labs name: roberta_jurisbert_class_tratados_internacionales_sistema_universal date: 2022-05-20 tags: [roberta, ner, text_classification, es, open_source] task: Text Classification language: es edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `jurisbert-class-tratados-internacionales-sistema-universal ` is a Spanish model orginally trained by `hackathon-pln-es`. ## Predicted Entities `Convención sobre la Eliminación de todas las formas de Discriminación contra la Mujer`, `Convención sobre los Derechos de las Personas con Discapacidad`, `Convención Internacional Sobre la Eliminación de Todas las Formas de Discriminación Racial`, `Convención contra la Tortura y otros Tratos o Penas Crueles, Inhumanos o Degradantes`, `Convención Internacional sobre la Protección de los Derechos de todos los Trabajadores Migratorios y de sus Familias`, `Convención de los Derechos del Niño`, `Pacto Internacional de Derechos Económicos, Sociales y Culturales`, `Pacto Internacional de Derechos Civiles y Políticos` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_jurisbert_class_tratados_internacionales_sistema_universal_es_3.4.4_3.0_1653050297872.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_jurisbert_class_tratados_internacionales_sistema_universal_es_3.4.4_3.0_1653050297872.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_jurisbert_class_tratados_internacionales_sistema_universal| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://huggingface.co/hackathon-pln-es/jurisbert-class-tratados-internacionales-sistema-universal --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-few-shot-k-32-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0_en_4.3.0_3.0_1674221604163.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0_en_4.3.0_3.0_1674221604163.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_few_shot_k_32_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-large-few-shot-k-32-finetuned-squad-seed-0 --- layout: model title: Pipeline to Detect PHI for Deidentification (Enriched) author: John Snow Labs name: ner_deid_enriched_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, enriched, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_enriched](https://nlp.johnsnowlabs.com/2021/03/31/ner_deid_enriched_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_pipeline_en_3.4.1_3.0_1647867856711.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_pipeline_en_3.4.1_3.0_1647867856711.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_enriched_pipeline", "en", "clinical/models") pipeline.annotate("HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Remedy City Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Remedy City Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_enriched_pipeline", "en", "clinical/models") pipeline.annotate("HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Remedy City Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Remedy City Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid_enriched.pipeline").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Remedy City Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Remedy City Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""") ```

## Results ```bash +--------------------+---------+ |chunk |ner_label| +--------------------+---------+ |Smith |PATIENT | |VA Hospital |HOSPITAL | |Remedy City Hospital|HOSPITAL | |02/04/2003 |DATE | |Smith |PATIENT | |Remedy City Hospital|HOSPITAL | |Smith |PATIENT | |Smith |PATIENT | |Hart |DOCTOR | |Smith |PATIENT | |02/07/2003 |DATE | +--------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_enriched_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Word2Vec Embeddings in Estonian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, et, open_source] task: Embeddings language: et edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_et_3.4.1_3.0_1647370411777.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_et_3.4.1_3.0_1647370411777.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Results +---------------+--------+ |chunks |entities| +---------------+--------+ |16 أبريل 2000 |DATE | |ليلى حسن |PATIENT | |789، |ZIP | |جدة |CITY | |54321 |ZIP | |المملكة العربية|CITY | |السعودية |COUNTRY | |النور |HOSPITAL| |أميرة أحمد |DOCTOR | |ليلى |PATIENT | |35 |AGE | +---------------+--------+ {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|ar| |Size:|1.2 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Turkish ElectraForQuestionAnswering model (from enelpi) author: John Snow Labs name: electra_qa_enelpi_squad date: 2022-06-22 tags: [tr, open_source, electra, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-tr-enelpi-squad-qa` is a Turkish model originally trained by `enelpi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_enelpi_squad_tr_4.0.0_3.0_1655921380832.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_enelpi_squad_tr_4.0.0_3.0_1655921380832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_enelpi_squad","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_enelpi_squad","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.squad.electra").predict("""Benim adım ne?|||"Benim adım Clara ve Berkeley'de yaşıyorum.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_enelpi_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|412.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/enelpi/electra-tr-enelpi-squad-qa --- layout: model title: Thai XlmRoBertaForQuestionAnswering (from wicharnkeisei) author: John Snow Labs name: xlm_roberta_qa_thai_xlm_roberta_base_squad2 date: 2022-06-23 tags: [th, open_source, question_answering, xlmroberta] task: Question Answering language: th edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `thai-xlm-roberta-base-squad2` is a Thai model originally trained by `wicharnkeisei`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_thai_xlm_roberta_base_squad2_th_4.0.0_3.0_1655988133743.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_thai_xlm_roberta_base_squad2_th_4.0.0_3.0_1655988133743.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_thai_xlm_roberta_base_squad2","th") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_thai_xlm_roberta_base_squad2","th") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("th.answer_question.squadv2.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_thai_xlm_roberta_base_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|th| |Size:|881.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/wicharnkeisei/thai-xlm-roberta-base-squad2 - https://github.com/iapp-technology/iapp-wiki-qa-dataset --- layout: model title: Pdf processing author: John Snow Labs name: pdf_processing date: 2023-01-03 tags: [en, licensed, ocr, pdf_processing] task: Document Pdf Processing language: en nav_key: models edition: Visual NLP 4.0.0 spark_version: 3.2.1 supported: true annotator: PdfProcessing article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model obtain the text of an input PDF document with a Tesseract pretrained model. Tesseract is an Optical Character Recognition (OCR) engine developed by Google. It is an open-source tool that can be used to recognize text in images and convert it into machine-readable text. The engine is based on a neural network architecture and uses machine learning algorithms to improve its accuracy over time. Tesseract has been trained on a variety of datasets to improve its recognition capabilities. These datasets include images of text in various languages and scripts, as well as images with different font styles, sizes, and orientations. The training process involves feeding the engine with a large number of images and their corresponding text, allowing the engine to learn the patterns and characteristics of different text styles. One of the most important datasets used in training Tesseract is the UNLV dataset, which contains over 400,000 images of text in different languages, scripts, and font styles. This dataset is widely used in the OCR community and has been instrumental in improving the accuracy of Tesseract. Other datasets that have been used in training Tesseract include the ICDAR dataset, the IIIT-HWS dataset, and the RRC-GV-WS dataset. In addition to these datasets, Tesseract also uses a technique called adaptive training, where the engine can continuously improve its recognition capabilities by learning from new images and text. This allows Tesseract to adapt to new text styles and languages, and improve its overall accuracy. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/PDF_TO_TEXT/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://github.com/JohnSnowLabs/spark-ocr-workshop/blob/master/tutorials/Certification_Trainings/2.1.Pdf_processing.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pdf_to_text = PdfToText() \ .setInputCol("content") \ .setOutputCol("text") \ .setSplitPage(True) \ .setExtractCoordinates(True) \ .setStoreSplittedPdf(True) pdf_to_image = PdfToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setKeepInput(True) ocr = ImageToText() \ .setInputCol("image") \ .setOutputCol("text") \ .setConfidenceThreshold(60) pipeline = PipelineModel(stages=[ pdf_to_text, pdf_to_image, ocr ]) pdf_path = pkg_resources.resource_filename('sparkocr', 'resources/ocr/pdfs/*.pdf') pdf_example_df = spark.read.format("binaryFile").load(pdf_path).cache() result = pipeline().transform(pdf_example_df).cache() ``` ```scala val pdf_to_text = new PdfToText() .setInputCol("content") .setOutputCol("text") .setSplitPage(True) .setExtractCoordinates(True) .setStoreSplittedPdf(True) val pdf_to_image = new PdfToImage() .setInputCol("content") .setOutputCol("image") .setKeepInput(True) val ocr = new ImageToText() .setInputCol("image") .setOutputCol("text") .setConfidenceThreshold(60) val pipeline = new PipelineModel().setStages(Array( pdf_to_text, pdf_to_image, ocr)) val pdf_path = pkg_resources.resource_filename("sparkocr", "resources/ocr/pdfs/*.pdf") val pdf_example_df = spark.read.format("binaryFile").load(pdf_path).cache() val result = pipeline().transform(pdf_example_df).cache() ```

## Example ### Input: Representation of images 2 and 3 of the example: ![Screenshot](/assets/images/examples_ocr/image7.png) ## Output text ```bash +--------------------+-------------------+------+--------------------+--------------------+-----------------+------------------+--------------------+--------------------+-----------+-------+-----------+----------------+---------+ | path| modificationTime|length| text| positions| height_dimension| width_dimension| content| image|total_pages|pagenum|documentnum| confidence|exception| +--------------------+-------------------+------+--------------------+--------------------+-----------------+------------------+--------------------+--------------------+-----------+-------+-----------+----------------+---------+ |file:/Users/nmeln...|2022-07-14 15:38:51|693743|Patient Nam\nFina...|[{[{Patient Nam\n...|1587.780029296875|1205.8299560546875|[25 50 44 46 2D 3...|{file:/Users/nmel...| 1| 0| 0|81.2276874118381| null| |file:/Users/nmeln...|2022-07-14 15:38:51|693743|Patient Name\nFin...|[{[{Patient Name\...|1583.780029296875|1217.8299560546875|[25 50 44 46 2D 3...|{file:/Users/nmel...| 1| 0| 0|78.5234429732613| null| |file:/Users/nmeln...|2022-07-14 15:38:51| 70556|Alexandria is the...|[{[{A, 0, 72.024,...| 792.0| 612.0|[25 50 44 46 2D 3...| null| null| 0| 0| null| null| |file:/Users/nmeln...|2022-07-14 15:38:51| 70556|Alexandria was fo...|[{[{A, 1, 72.024,...| 792.0| 612.0|[25 50 44 46 2D 3...| null| null| 0| 0| null| null| |file:/Users/nmeln...|2022-07-14 15:38:51| 11601|8 i , . ! \n9 i ,...|[{[{8, 0, 72.0604...| 843.0| 596.0|[25 50 44 46 2D 3...| null| null| 0| 0| null| null| +--------------------+-------------------+------+--------------------+--------------------+-----------------+------------------+--------------------+--------------------+-----------+-------+-----------+----------------+---------+ ``` ```bash text 0 Patient Nam Financial Numbe Random Hospital Date of Birth Patient Location Chief Complaint Shortness of breath History of Present Illness Patient is an 84-year-old male wilh a past medical history of hypertension, HFpEF last known EF 55%, mild to moderate TA, pulmonary hypertension, permanent atrial fibrillation on Eliquis, history of GI blesd, CK-M8, and anemia who presents with full weeks oi ccneralized fatigue and fecling unwell. He also notes some shortness oi Breath and worsening dyspnea willy minimal exerlion. His major complaints are shoulder and joint pains. diffusely. He also complains of "bone pain’. He denics having any fevers or cnills. e demes having any chest pain, palpitalicns, He denies any worse extremity swelling than his baseline. He states he’s been compliant with his mcdications. Although he stales he ran out of his Eliquis & few weeks ago. He denies having any blood in his stools or mc!ena, although he does take iron pills and states his stools arc irequently black. His hemoglobin Is al baseline. Twelve-lead EKG showing atrial fibrillation, RBBB, LAFB, PVC. Chest x-ray showing new small right creater than left pleural effusions with mild pulmonary vascular congestion. BNP increased to 2800, up fram 1900. Tropoain 0.03. Renal function at baseline. Hemoaglopin at baseline. She normally takes 80 mq of oral Lasix daily. He was given 80 mg of IV Lasix in the ED. He is currently net negative close to 1 L. He is still on 2 L nasal cannula. ' Ss 5 A 10 system roview af systems was completed and negative except as documented in HPI. Physical Exam Vitals & Measurements T: 36.8 °C (Oral) TMIN: 36.8 "C (Oral) TMAX: 37.0 °C (Oral) HR: 54 RR: 7 BP: 140/63 WT: 100.3 KG Pulse Ox: 100 % Oxygen: 2 L'min via Nasal Cannula GENERAL: no acute distress HEAD: normecephalic EYES‘EARS‘NOSE/THAOAT: nupils are equal. normal oropharynx NECK: normal inspection RESPIRATORY: no respiratory distress, no rales on my exam CARDIOVASCULAR: irregular. brady. no murmurs, rubs or galleps ABDOMEN: soft, non-tendes EXTREMITIES: Bilateral chronic venous stasis changes NEUROLOGIC: alert and osieniec x 3. no gross motar or sensory deficils AssessmenvPlan Acute on chronic diastolic CHF (congestive heart failure) Acute on chronic diastolic heart failure exacerbation. Small pleural effusions dilaterally with mild pulmonary vascular congesiion on chest x-ray, slighi elevation in BNR. We'll continue 1 more day af IV diuresis with 80 mg IV Lasix. He may have had a viral infection which precipilated this. We'll add Tylenol jor his joint paias. Continue atenclol and chiorthalidone. AF - Atrial fibrillation Permanent atrial fibrillation. Rates bradycardic in the &0s. Continue atenolol with hola parameters. Coniinue Eliquis for stroke prevention. No evidence oj bleeding, hemog'abin at baseline. Printed: 7/17/2017 13:01 EDT Page 16 of 42 Arincitis CHF - Congestive heart failure Chronic kidney disease Chronic venous insufficiency Edema GI bleeding Glaucoma Goul Hypertension Peptic ulcer Peripheral ncuropathy Peripheral vascular disease Pulmonary hypertension Tricuspid regurgitation Historical No qualifying data Procedure/Surgical History duodenal resection, duodenojcjunostomy. small bowel enterolomy, removal of foreign object and repair oi enterotomy (05/2 1/20 14), colonoscopy (12/10/2013), egd (1209/2013), H/O endoscopy (07/2013), H’O colonoscopy (03/2013), pilonidal cyst removal at base of spine (1981), laser eye surgery ior glaucoma. lesions on small intestine closed up. Home Medications Home allopurinol 300 mg oral tablet, 300 MG= 1 TAB, PO. Daily atenolol 25 mg oral tablet, 25 MG= 1 TAB, PO, Daily chtorthalidone 25 mg oral tablet, 23 MG= 1 TAB, PO, MVE Combigan 0.2%-0.5% ophthalmic solution, 1 DROP, Both Eyes, Q12H Eliquis 5 mg oral lablet, 5 MG= 1 TAB, PO, BID lerrous sulfate 925 mg (65 nig elemental iron) oral tablet, 325 MG= 1 TAB, PO, Daily Lasix 80 mg oral tabic:. 80 MG= | TAB. PO, BID omeprazole 20 mg oral delayed scicasc capsule, 20 MG= 1 CAP, PO, BID Percocei 5/325 oral tablet. | TAB, PO. QAM potassium chloride 20 mEq oral tablet, extended release, 20 MEO= 1 TAB, PO, Daily sertraline 50 mg oral tablet, 75 MG= 1,5 TAB, PQ. Daiiy triamcinolone 0.71% lopical cream, 1 APP, Topical, Daily lriamcmnolone 0.1% lopical ominient, 1 APP. Topical, Daily PowerChart 1 Patient Name Financial Number Date of Girth Patient Location Random Hospital H & P Anemia Vitamin D2 50,000 intl units (1.25 ma) oral ALBASeRne capsule, 1 TAS, PO, Veexly-Tue Arthritis Allergies Tylenol for pain. Patient also takes Percocet alt home, will add this cn. Chronic kidney disease AY baseline. Monitor while divresing. Hypertension Blood pressures within tolerable ranges. Pulmonary hypertension Tricuspid regurgitation Mild-to-moderaie on echocardiogram last year sholliisn (cout) sulfa drug (maculopapular rash) Social History Ever Smoked tobacco: Former Smoker Alcohol use - frequency; None Drug use: Never Lab Results O7/16/9 7 05:30 to O7/16/17 05:30 Attending physician note-the patient was interviewed and examined. The appropriatc information in power chart was reviewed. The patient was discussed wilh Dr, Persad. 143 1L 981H 26? Patient may have @ mild degree of heart failure. He and his wife were more concernes with ee Ins peripheral edema. He has underlying renal insufficiency as well. We'll try to diurese him to his “dry" weight. We will then try to adjust his medications to kcep him within & narrow range of [hat weight. We will stop his atenolol this point since he is relatively bradycardic anc observe his heart rate on the cardiac monitor. He will progress with his care and aclivily as tolerated. 102 07/16/17 05:30 to O7/ 16/17 05:30 05:30 GLU 102 mg/dL Printed: 7/1 7/2017 13:01 EDT Page 17 of 42 NA K CL TOTAL COZ BUN CRT ANION GAP CA CBC with diff WBC HGB HCT RBC MCV MICH MCHC RDW MPV 143 MMOL/L 3.6 MMOL/L 98 MMOL/L 40 MMOL/L 26 mg/dL. 1.23 mg/dL 5 7.9 mg/dL 07/16/17 05:30 3.4/ nl 10.1 G/DL 32.4 %o 3.41 /PL 95.0 FL 29.6 pg 31.2 % 15,9 %o 10.7 FL PowerChart 2 Alexandria is the second-largest city in Egypt and a major economic centre, extending about 32 km (20 mi) along the coast of the Mediterranean Sea in the north central part of the country. Its low elevation on the Nile delta makes it highly vulnerable to rising sea levels. Alexandria is an important industrial center because of its natural gas and oil pipelines from Suez. Alexandria is also a popular tourist destination. 3 Alexandria was founded around a small, ancient Egyptian town c. 332 BC by Alexander the Great,[4] king of Macedon and leader of the Greek League of Corinth, during his conquest of the Achaemenid Empire. Alexandria became an important center of Hellenistic civilization and remained the capital of Ptolemaic Egypt and Roman and Byzantine Egypt for almost 1,000 years, until the Muslim conquest of Egypt in AD 641, when a new capital was founded at Fustat (later absorbed into Cairo). Hellenistic Alexandria was best known for the Lighthouse of Alexandria (Pharos), one of the Seven Wonders of the Ancient World; its Great Library (the largest in the ancient world); and the Necropolis, one of the Seven Wonders of the Middle Ages. Alexandria was at one time the second most powerful city of the ancient Mediterranean region, after Rome. Ongoing maritime archaeology in the harbor of Alexandria, which began in 1994, is revealing details of Alexandria both before the arrival of Alexander, when a city named Rhacotis existed there, and during the Ptolemaic dynasty. 4 8 i , . ! 9 i , . ! 10 i , . ! 11 i , . ! 12 i , . ! 13 i , . ! 14 i , . !``` ``` ## Model Information {:.table-model} |---|---| |Model Name:|pdf_processing| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Swedish BertForTokenClassification Cased model (from hkaraoguz) author: John Snow Labs name: bert_token_classifier_swedish_ner date: 2022-11-29 tags: [sv, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: sv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BERT_swedish-ner` is a Swedish model originally trained by `hkaraoguz`. ## Predicted Entities `PER`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_swedish_ner_sv_4.2.4_3.0_1669733190021.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_swedish_ner_sv_4.2.4_3.0_1669733190021.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_swedish_ner","sv") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_swedish_ner","sv") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_swedish_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|sv| |Size:|465.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/hkaraoguz/BERT_swedish-ner - https://paperswithcode.com/sota?task=Token+Classification&dataset=wikiann --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0_en_4.3.0_3.0_1674215332011.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0_en_4.3.0_3.0_1674215332011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_512_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|433.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-0 --- layout: model title: Word2Vec Embeddings in Western Panjabi (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, pnb, open_source] task: Embeddings language: pnb edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pnb_3.4.1_3.0_1647467596944.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pnb_3.4.1_3.0_1647467596944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2t_r_wav2vec2_s37', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2t_r_wav2vec2_s37", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2t_r_wav2vec2_s37| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Assignments Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_assignments_bert date: 2023-03-05 tags: [en, legal, classification, clauses, assignments, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Assignments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Assignments`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_assignments_bert_en_1.0.0_3.0_1678050646575.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_assignments_bert_en_1.0.0_3.0_1678050646575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_assignments_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Assignments]| |[Other]| |[Other]| |[Assignments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_assignments_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Assignments 0.93 0.94 0.93 157 Other 0.95 0.94 0.94 188 accuracy - - 0.94 345 macro-avg 0.94 0.94 0.94 345 weighted-avg 0.94 0.94 0.94 345 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Fijian author: John Snow Labs name: opus_mt_en_fj date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, fj, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `fj` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_fj_xx_2.7.0_2.4_1609164632179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_fj_xx_2.7.0_2.4_1609164632179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_fj| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Japanese Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_base_japanese_unidic_luw_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, ja, open_source] task: Part of Speech Tagging language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-unidic-luw-upos` is a Japanese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_japanese_unidic_luw_upos_ja_3.4.2_3.0_1652092034890.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_japanese_unidic_luw_upos_ja_3.4.2_3.0_1652092034890.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_japanese_unidic_luw_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ja| |Size:|415.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-base-japanese-unidic-luw-upos - https://universaldependencies.org/u/pos/ - https://pypi.org/project/fugashi - https://pypi.org/project/unidic-lite - https://pypi.org/project/pytokenizations - http://id.nii.ac.jp/1001/00216223/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from FabianWillner) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_trivia date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-triviaqa` is a English model originally trained by `FabianWillner`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_trivia_en_4.3.0_3.0_1672773828302.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_trivia_en_4.3.0_3.0_1672773828302.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_trivia","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_trivia","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_trivia| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/FabianWillner/distilbert-base-uncased-finetuned-triviaqa --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-0` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1654189490798.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0_en_4.0.0_3.0_1654189490798.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_1024d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|390.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-0 --- layout: model title: Part of Speech for Afrikaans author: John Snow Labs name: pos_afribooms date: 2021-04-06 tags: [pos, open_source, af] task: Part of Speech Tagging language: af edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_afribooms_af_3.0.0_3.0_1617749039095.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_afribooms_af_3.0.0_3.0_1617749039095.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_afribooms", "af")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_afribooms", "af") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer ,pos)) val data = Seq("Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .""] token_df = nlu.load('af.pos.afribooms').predict(text) token_df ```

## Results ```bash +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ |text |result | +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ |Die kodes wat gebruik word , moet duidelik en verstaanbaar vir leerders en ouers wees .|[DET, NOUN, PRON, VERB, AUX, PUNCT, AUX, ADJ, CCONJ, ADJ, ADP, NOUN, CCONJ, NOUN, AUX, PUNCT]| +---------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_afribooms| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|af| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.59 | 0.65 | 0.62 | 665 | | ADP | 0.76 | 0.79 | 0.77 | 1299 | | ADV | 0.72 | 0.69 | 0.71 | 523 | | AUX | 0.86 | 0.83 | 0.84 | 663 | | CCONJ | 0.70 | 0.71 | 0.70 | 380 | | DET | 0.84 | 0.70 | 0.76 | 1014 | | NOUN | 0.68 | 0.72 | 0.70 | 2025 | | NUM | 0.89 | 0.81 | 0.85 | 42 | | PART | 0.67 | 0.68 | 0.67 | 322 | | PRON | 0.87 | 0.87 | 0.87 | 794 | | PROPN | 0.87 | 0.67 | 0.75 | 156 | | PUNCT | 0.68 | 0.70 | 0.69 | 877 | | SCONJ | 0.82 | 0.85 | 0.83 | 210 | | SYM | 0.86 | 0.88 | 0.87 | 142 | | VERB | 0.69 | 0.71 | 0.70 | 889 | | X | 0.24 | 0.14 | 0.18 | 64 | | accuracy | | | 0.73 | 10065 | | macro avg | 0.73 | 0.71 | 0.72 | 10065 | | weighted avg | 0.74 | 0.73 | 0.73 | 10065 | ``` --- layout: model title: Social Determinants of Health (clinical_large) author: John Snow Labs name: ner_sdoh_emb_clinical_large_wip date: 2023-04-12 tags: [en, clinical_large, social_determinants, public_health, ner, sdoh, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts terminology related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Access_To_Care`, `Age`, `Alcohol`, `Chidhood_Event`, `Communicable_Disease`, `Community_Safety`, `Diet`, `Disability`, `Eating_Disorder`, `Education`, `Employment`, `Environmental_Condition`, `Exercise`, `Family_Member`, `Financial_Status`, `Food_Insecurity`, `Gender`, `Geographic_Entity`, `Healthcare_Institution`, `Housing`, `Hyperlipidemia`, `Hypertension`, `Income`, `Insurance_Status`, `Language`, `Legal_Issues`, `Marital_Status`, `Mental_Health`, `Obesity`, `Other_Disease`, `Other_SDoH_Keywords`, `Population_Group`, `Quality_Of_Life`, `Race_Ethnicity`, `Sexual_Activity`, `Sexual_Orientation`, `Smoking`, `Social_Exclusion`, `Social_Support`, `Spiritual_Beliefs`, `Substance_Duration`, `Substance_Frequency`, `Substance_Quantity`, `Substance_Use`, `Transportation`, `Violence_Or_Abuse` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_NER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_emb_clinical_large_wip_en_4.3.2_3.2_1681303888925.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_emb_clinical_large_wip_en_4.3.2_3.2_1681303888925.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_emb_clinical_large_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = [["Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week."]] data = spark.createDataFrame(sample_texts).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_emb_clinical_large_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("Smith is a 55 years old, divorced Mexcian American woman with financial problems. She speaks spanish. She lives in an apartment. She has been struggling with diabetes for the past 10 years and has recently been experiencing frequent hospitalizations due to uncontrolled blood sugar levels. Smith works as a cleaning assistant and does not have access to health insurance or paid sick leave. She has a son student at college. Pt with likely long-standing depression. She is aware she needs rehab. Pt reprots having her catholic faith as a means of support as well. She has long history of etoh abuse, beginning in her teens. She reports she has been a daily drinker for 30 years, most recently drinking beer daily. She smokes a pack of cigarettes a day. She had DUI back in April and was due to be in court this week.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +------------------+-----+---+-------------------+ |chunk |begin|end|ner_label | +------------------+-----+---+-------------------+ |55 years old |11 |22 |Age | |divorced |25 |32 |Marital_Status | |Mexcian |34 |40 |Race_Ethnicity | |American |42 |49 |Race_Ethnicity | |woman |51 |55 |Gender | |financial problems|62 |79 |Financial_Status | |She |82 |84 |Gender | |spanish |93 |99 |Language | |She |102 |104|Gender | |apartment |118 |126|Housing | |She |129 |131|Gender | |diabetes |158 |165|Other_Disease | |hospitalizations |233 |248|Other_SDoH_Keywords| |cleaning assistant|307 |324|Employment | |health insurance |354 |369|Insurance_Status | |She |391 |393|Gender | |son |401 |403|Family_Member | |student |405 |411|Education | |college |416 |422|Education | |depression |454 |463|Mental_Health | |She |466 |468|Gender | |she |479 |481|Gender | |rehab |489 |493|Access_To_Care | |her |514 |516|Gender | |catholic faith |518 |531|Spiritual_Beliefs | |support |547 |553|Social_Support | |She |565 |567|Gender | |etoh abuse |589 |598|Alcohol | |her |614 |616|Gender | |teens |618 |622|Age | |She |625 |627|Gender | |she |637 |639|Gender | |daily |652 |656|Substance_Quantity | |drinker |658 |664|Alcohol | |30 years |670 |677|Substance_Duration | |drinking beer |694 |706|Alcohol | |daily |708 |712|Substance_Frequency| |She |715 |717|Gender | |smokes |719 |724|Smoking | |a pack |726 |731|Substance_Quantity | |cigarettes |736 |745|Smoking | |a day |747 |751|Substance_Frequency| |She |754 |756|Gender | |DUI |762 |764|Legal_Issues | +------------------+-----+---+-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_emb_clinical_large_wip| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.0 MB| |Dependencies:|embeddings_clinical_large| ## References Internal SHOP Project ## Benchmarking ```bash label precision recall f1-score support Employment 0.94 0.96 0.95 2075 Social_Support 0.91 0.90 0.90 658 Other_SDoH_Keywords 0.82 0.87 0.85 259 Healthcare_Institution 0.99 0.95 0.97 781 Alcohol 0.96 0.97 0.96 258 Gender 0.99 0.99 0.99 4957 Other_Disease 0.89 0.94 0.91 583 Access_To_Care 0.86 0.88 0.87 520 Mental_Health 0.89 0.81 0.85 494 Age 0.92 0.96 0.94 433 Marital_Status 1.00 1.00 1.00 92 Substance_Quantity 0.88 0.86 0.87 58 Substance_Use 0.91 0.97 0.94 192 Family_Member 0.97 0.99 0.98 2094 Financial_Status 0.86 0.65 0.74 124 Race_Ethnicity 0.93 0.93 0.93 27 Insurance_Status 0.93 0.87 0.90 85 Spiritual_Beliefs 0.86 0.81 0.83 52 Housing 0.88 0.85 0.87 400 Geographic_Entity 0.86 0.88 0.87 113 Disability 0.93 0.93 0.93 44 Quality_Of_Life 0.89 0.75 0.81 67 Income 0.89 0.77 0.83 31 Education 0.85 0.88 0.86 58 Transportation 0.86 0.89 0.88 57 Legal_Issues 0.72 0.91 0.80 47 Smoking 0.98 0.97 0.98 66 Substance_Frequency 0.93 0.75 0.83 57 Hypertension 1.00 1.00 1.00 21 Violence_Or_Abuse 0.83 0.62 0.71 63 Exercise 0.96 0.88 0.92 57 Diet 0.95 0.87 0.91 70 Sexual_Orientation 0.68 1.00 0.81 13 Language 0.89 0.73 0.80 22 Social_Exclusion 0.96 0.90 0.93 29 Substance_Duration 0.75 0.85 0.80 39 Communicable_Disease 1.00 0.84 0.91 31 Chidhood_Event 0.88 0.61 0.72 23 Community_Safety 0.95 0.93 0.94 44 Population_Group 0.89 0.62 0.73 13 Hyperlipidemia 0.78 1.00 0.88 7 Food_Insecurity 1.00 0.93 0.96 29 Eating_Disorder 0.67 0.92 0.77 13 Sexual_Activity 0.84 0.90 0.87 29 Environmental_Condition 1.00 1.00 1.00 20 Obesity 1.00 1.00 1.00 12 micro-avg 0.95 0.95 0.95 15217 macro-avg 0.90 0.88 0.88 15217 weighted-avg 0.95 0.95 0.95 15217 ``` --- layout: model title: Legal Indemnification Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_indemnification_agreement date: 2022-11-10 tags: [en, legal, licensed, classification, indemnification] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_indemnification_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `indemnification-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `indemnification-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_agreement_en_1.0.0_3.0_1668108212729.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnification_agreement_en_1.0.0_3.0_1668108212729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[indemnification-agreement]| |[other]| |[other]| |[indemnification-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnification_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support indemnification-agreement 1.00 1.00 1.00 31 other 1.00 1.00 1.00 85 accuracy - - 1.00 116 macro-avg 1.00 1.00 1.00 116 weighted-avg 1.00 1.00 1.00 116 ``` --- layout: model title: English DistilBertForMaskedLM Cased model (from joaogante) author: John Snow Labs name: distilbert_embeddings_test_text date: 2022-12-12 tags: [en, open_source, distilbert_embeddings, distilbertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `test_text` is a English model originally trained by `joaogante`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_test_text_en_4.2.4_3.0_1670865040247.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_test_text_en_4.2.4_3.0_1670865040247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_test_text","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(False) pipeline = Pipeline(stages=[documentAssembler, tokenizer, distilbert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_test_text","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(false) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, distilbert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_test_text| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| ## References - https://huggingface.co/joaogante/test_text - https://arxiv.org/abs/1910.01108 - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English BertForQuestionAnswering model (from youngjae) author: John Snow Labs name: bert_qa_youngjae_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `youngjae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_youngjae_bert_finetuned_squad_en_4.0.0_3.0_1654535781311.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_youngjae_bert_finetuned_squad_en_4.0.0_3.0_1654535781311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_youngjae_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_youngjae_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_youngjae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_youngjae_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/youngjae/bert-finetuned-squad --- layout: model title: Spanish Bert Embeddings (Base, Question, Squades) author: John Snow Labs name: bert_embeddings_dpr_spanish_question_encoder_squades_base date: 2022-04-11 tags: [bert, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `dpr-spanish-question_encoder-squades-base` is a Spanish model orginally trained by `IIC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_dpr_spanish_question_encoder_squades_base_es_3.4.2_3.0_1649671254424.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_dpr_spanish_question_encoder_squades_base_es_3.4.2_3.0_1649671254424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_dpr_spanish_question_encoder_squades_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|412.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/IIC/dpr-spanish-question_encoder-squades-base - https://arxiv.org/abs/2004.04906 - https://github.com/facebookresearch/DPR - https://paperswithcode.com/sota?task=text+similarity&dataset=squad_es --- layout: model title: English AlbertForQuestionAnswering model (from saburbutt) author: John Snow Labs name: albert_qa_generic date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qa-generic` is a English model originally trained by `saburbutt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_generic_en_4.0.0_3.0_1656064418789.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_generic_en_4.0.0_3.0_1656064418789.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_generic","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_generic","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.albert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_generic| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|771.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saburbutt/testing --- layout: model title: Translate English to Portuguese-based languages Pipeline author: John Snow Labs name: translate_en_cpp date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, cpp, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `cpp` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_cpp_xx_2.7.0_2.4_1609699630268.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_cpp_xx_2.7.0_2.4_1609699630268.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_cpp", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_cpp", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.cpp').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_cpp| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Czech to English Pipeline author: John Snow Labs name: translate_cs_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, cs, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `cs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_cs_en_xx_2.7.0_2.4_1609690011748.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_cs_en_xx_2.7.0_2.4_1609690011748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_cs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_cs_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.cs.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_cs_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Breton Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:34:00 +0800 task: Lemmatization language: br edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, br] supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_br_2.5.5_2.4_1596054394143.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_br_2.5.5_2.4_1596054394143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "br") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Distaolit dimp hon dleoù evel m'hor bo ivez distaolet d'hon dleourion.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "br") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Distaolit dimp hon dleoù evel m"hor bo ivez distaolet d"hon dleourion.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Distaolit dimp hon dleoù evel m'hor bo ivez distaolet d'hon dleourion."""] lemma_df = nlu.load('br.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=8, result='Distaolit', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=10, end=13, result='_', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=15, end=17, result='kaout', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=19, end=23, result='dleoù', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=25, end=28, result='evel', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|br| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Word2Vec Embeddings in Punjabi (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, pa, open_source] task: Embeddings language: pa edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pa_3.4.1_3.0_1647294419824.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pa_3.4.1_3.0_1647294419824.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_youngjae_bert_finetuned_squad_accelerate","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_youngjae_bert_finetuned_squad_accelerate","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.accelerate.by_youngjae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_youngjae_bert_finetuned_squad_accelerate| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/youngjae/bert-finetuned-squad-accelerate --- layout: model title: Legal Consideration Clause Binary Classifier author: John Snow Labs name: legclf_consideration_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `consideration` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `consideration` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_consideration_clause_en_1.0.0_3.2_1660122285073.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_consideration_clause_en_1.0.0_3.2_1660122285073.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_consideration_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[consideration]| |[other]| |[other]| |[consideration]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_consideration_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support consideration 0.73 0.65 0.69 34 other 0.91 0.94 0.92 128 accuracy - - 0.88 162 macro-avg 0.82 0.79 0.81 162 weighted-avg 0.87 0.88 0.87 162 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from hfl) author: John Snow Labs name: bert_embeddings_chinese_wwm date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-bert-wwm` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_wwm_zh_4.2.4_3.0_1670020906241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_wwm_zh_4.2.4_3.0_1670020906241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_wwm","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_wwm","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_wwm| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-bert-wwm - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Context Spell Checker Pipeline for English author: John Snow Labs name: spellcheck_dl_pipeline date: 2022-06-19 tags: [spellcheck, spell, spellcheck_pipeline, spelling_corrector, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained spellchecker pipeline is built on the top of [spellcheck_dl](https://nlp.johnsnowlabs.com/2022/04/02/spellcheck_dl_en_2_4.html) model. This pipeline is for PySpark 2.4.x users with SparkNLP 3.4.2 and above. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_4.0.0_3.0_1655653892700.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_4.0.0_3.0_1655653892700.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash [{'checked': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'weather', '.'], 'document': ['During the summer we have the best ueather.'], 'token': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'ueather', '.']}, {'checked': ['I', 'have', 'a', 'black', 'leather', 'jacket', ',', 'so', 'nice', '.'], 'document': ['I have a black ueather jacket, so nice.'], 'token': ['I', 'have', 'a', 'black', 'ueather', 'jacket', ',', 'so', 'nice', '.']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_dl_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|99.8 MB| ## Included Models - DocumentAssembler - TokenizerModel - ContextSpellCheckerModel --- layout: model title: Translate Ndonga to English Pipeline author: John Snow Labs name: translate_ng_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ng, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ng` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ng_en_xx_2.7.0_2.4_1609691829310.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ng_en_xx_2.7.0_2.4_1609691829310.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ng_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ng_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ng.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ng_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_finetuned_audio_transcriber TFWav2Vec2ForCTC from mfreihaut author: John Snow Labs name: pipeline_asr_finetuned_audio_transcriber date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_finetuned_audio_transcriber` is a English model originally trained by mfreihaut. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_finetuned_audio_transcriber_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_finetuned_audio_transcriber_en_4.2.0_3.0_1664122195152.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_finetuned_audio_transcriber_en_4.2.0_3.0_1664122195152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_finetuned_audio_transcriber', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_finetuned_audio_transcriber", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_finetuned_audio_transcriber| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Arabic Part of Speech Tagger (CA-Classical Arabic dataset, Gulf Arabic POS) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_ca_pos_glf date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-ca-pos-glf` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_glf_ar_3.4.2_3.0_1650993546516.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_glf_ar_3.4.2_3.0_1650993546516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_ca_pos_glf| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-ca-pos-glf - https://camel.abudhabi.nyu.edu/annotated-gumar-corpus/ - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: Spanish DistilBertForQuestionAnswering Base Uncased model (from CenIA) author: John Snow Labs name: distilbert_qa_distillbert_base_uncased_finetuned_ml date: 2023-01-03 tags: [es, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distillbert-base-spanish-uncased-finetuned-qa-mlqa` is a Spanish model originally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_uncased_finetuned_ml_es_4.3.0_3.0_1672774747112.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_uncased_finetuned_ml_es_4.3.0_3.0_1672774747112.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_uncased_finetuned_ml","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_uncased_finetuned_ml","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distillbert_base_uncased_finetuned_ml| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|250.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/distillbert-base-spanish-uncased-finetuned-qa-mlqa --- layout: model title: Detect PHI for Deidentification (Augmented) author: John Snow Labs name: ner_deid_synthetic date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deidentification NER (Augmented) is a Named Entity Recognition model that annotates text to find protected health information that may need to be deidentified. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_synthetic_en_3.0.0_3.0_1617209707564.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_synthetic_en_3.0.0_3.0_1617209707564.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained("ner_deid_synthetic","en","clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same. ']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_deid_synthetic", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same. """).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.synthetic").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital, Dr. John Green (2347165768). He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same. """) ```

## Results ```bash +---------------+---------+ |chunk |ner_label| +---------------+---------+ |Smith |NAME | |VA Hospital |LOCATION | |John Green |NAME | |2347165768 |ID | |Day Hospital |LOCATION | |02/04/2003 |DATE | |Smith |NAME | |Day Hospital |LOCATION | |Smith |NAME | |Smith |NAME | |7 Ardmore Tower|LOCATION | |Hart |NAME | |Smith |NAME | |02/07/2003 |DATE | +---------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_synthetic| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on augmented synthetic PHI data other links (n2c2 2014: De-identification and Heart Disease Risk Factors Challenge datasets with embeddings_clinical https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/) ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | I-NAME | 1096 | 47 | 80 | 0.95888 | 0.931973 | 0.945235 | | 1 | I-CONTACT | 93 | 0 | 4 | 1 | 0.958763 | 0.978947 | | 2 | I-AGE | 3 | 1 | 6 | 0.75 | 0.333333 | 0.461538 | | 3 | B-DATE | 2078 | 42 | 52 | 0.980189 | 0.975587 | 0.977882 | | 4 | I-DATE | 474 | 39 | 25 | 0.923977 | 0.9499 | 0.936759 | | 5 | I-LOCATION | 755 | 68 | 76 | 0.917375 | 0.908544 | 0.912938 | | 6 | I-PROFESSION | 78 | 8 | 9 | 0.906977 | 0.896552 | 0.901734 | | 7 | B-NAME | 1182 | 101 | 36 | 0.921278 | 0.970443 | 0.945222 | | 8 | B-AGE | 259 | 10 | 11 | 0.962825 | 0.959259 | 0.961039 | | 9 | B-ID | 146 | 8 | 11 | 0.948052 | 0.929936 | 0.938907 | | 10 | B-PROFESSION | 76 | 9 | 21 | 0.894118 | 0.783505 | 0.835165 | | 11 | B-LOCATION | 556 | 87 | 71 | 0.864697 | 0.886762 | 0.875591 | | 12 | I-ID | 64 | 8 | 3 | 0.888889 | 0.955224 | 0.920863 | | 13 | B-CONTACT | 40 | 7 | 5 | 0.851064 | 0.888889 | 0.869565 | | 14 | Macro-average | 6900 | 435 | 410 | 0.912023 | 0.880619 | 0.896046 | | 15 | Micro-average | 6900 | 435 | 410 | 0.940695 | 0.943912 | 0.942301 | ``` --- layout: model title: English BertForQuestionAnswering model (from howey) author: John Snow Labs name: bert_qa_howey_bert_large_uncased_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-squad` is a English model orginally trained by `howey`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_howey_bert_large_uncased_squad_en_4.0.0_3.0_1654536567405.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_howey_bert_large_uncased_squad_en_4.0.0_3.0_1654536567405.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_howey_bert_large_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_howey_bert_large_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.large_uncased.by_howey").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_howey_bert_large_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/howey/bert-large-uncased-squad --- layout: model title: Word2Vec Embeddings in Ossetic (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, os, open_source] task: Embeddings language: os edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_os_3.4.1_3.0_1647451060944.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_os_3.4.1_3.0_1647451060944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) spanClassifier = nlp.RoBertaForQuestionAnswering.pretrained("legqa_roberta_cuad_large","en", "legal/models") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = nlp.Pipeline().setStages([ documentAssembler, spanClassifier ]) text = """THIS CREDIT AGREEMENT is dated as of April 29, 2010, and is made by and among P.H. GLATFELTER COMPANY, a Pennsylvania corporation ( the "COMPANY") and certain of its subsidiaries. Identified on the signature pages hereto (each a "BORROWER" and collectively, the "BORROWERS"), each of the GUARANTORS (as hereinafter defined), the LENDERS (as hereinafter defined), PNC BANK, NATIONAL ASSOCIATION, in its capacity as agent for the Lenders under this Agreement (hereinafter referred to in such capacity as the "ADMINISTRATIVE AGENT"), and, for the limited purpose of public identification in trade tables, PNC CAPITAL MARKETS LLC and CITIZENS BANK OF PENNSYLVANIA, as joint arrangers and joint bookrunners, and CITIZENS BANK OF PENNSYLVANIA, as syndication agent.""".replace('\n',' ') question = ['"Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract"'] qt = [ [q,text] for q in questions ] example = spark.createDataFrame(qt).toDF("question", "context") result = pipeline.fit(example).transform(example) result.select('document_question.result', 'answer.result').show(truncate=False) ```

## Results ```bash ["Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract"]|[P . H . GLATFELTER COMPANY]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legqa_roberta_cuad_large| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|454.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References Squad, finetuned with CUAD-based Question/Answering --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223967265.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223967265.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_twostagetriplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|306.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_twostagetriplet_epochs_1_shard_1_squad2.0 --- layout: model title: English BertForTokenClassification Uncased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4_Modified_bluebert_pubmed_uncased_L_12_H_768_A_12 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4_Modified-bluebert_pubmed_uncased_L-12_H-768_A-12` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Modified_bluebert_pubmed_uncased_L_12_H_768_A_12_en_4.0.0_3.0_1657109104665.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4_Modified_bluebert_pubmed_uncased_L_12_H_768_A_12_en_4.0.0_3.0_1657109104665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4_Modified_bluebert_pubmed_uncased_L_12_H_768_A_12| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|407.6 MB| |Case sensitive:|false| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4_Modified-bluebert_pubmed_uncased_L-12_H-768_A-12 --- layout: model title: Japanese BertForQuestionAnswering Base Cased model (from chiba) author: John Snow Labs name: bert_qa_base_japanese_whole_word_masking_tes date: 2022-07-07 tags: [ja, open_source, bert, question_answering] task: Question Answering language: ja edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-japanese-whole-word-masking_test` is a Japanese model originally trained by `chiba`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_japanese_whole_word_masking_tes_ja_4.0.0_3.0_1657183050287.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_japanese_whole_word_masking_tes_ja_4.0.0_3.0_1657183050287.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_japanese_whole_word_masking_tes","ja") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_japanese_whole_word_masking_tes","ja") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_japanese_whole_word_masking_tes| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ja| |Size:|413.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/chiba/bert-base-japanese-whole-word-masking_test --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from rsvp-ai) author: John Snow Labs name: roberta_qa_bertserini_base date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertserini-roberta-base` is a English model originally trained by `rsvp-ai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_bertserini_base_en_4.3.0_3.0_1674209264464.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_bertserini_base_en_4.3.0_3.0_1674209264464.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bertserini_base","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bertserini_base","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_bertserini_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|449.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/rsvp-ai/bertserini-roberta-base --- layout: model title: English T5ForConditionalGeneration Small Cased model (from mtreviso) author: John Snow Labs name: t5_ct5_small_wiki_l2r date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ct5-small-en-wiki-l2r` is a English model originally trained by `mtreviso`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_ct5_small_wiki_l2r_en_4.3.0_3.0_1675100797165.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_ct5_small_wiki_l2r_en_4.3.0_3.0_1675100797165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ct5_small_wiki_l2r","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ct5_small_wiki_l2r","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_ct5_small_wiki_l2r| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|288.7 MB| ## References - https://huggingface.co/mtreviso/ct5-small-en-wiki-l2r - https://github.com/mtreviso/chunked-t5 --- layout: model title: Portuguese Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_bert_large_portuguese_cased date: 2022-04-11 tags: [bert, embeddings, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-portuguese-cased` is a Portuguese model orginally trained by `neuralmind`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_portuguese_cased_pt_3.4.2_3.0_1649673670038.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_portuguese_cased_pt_3.4.2_3.0_1649673670038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_portuguese_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pt| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/neuralmind/bert-large-portuguese-cased - https://github.com/neuralmind-ai/portuguese-bert/ --- layout: model title: Detect Entities Related to Cancer Diagnosis author: John Snow Labs name: ner_oncology_diagnosis date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to cancer diagnosis, such as Metastasis, Histological_Type or Invasion. Definitions of Predicted Entities: - `Adenopathy`: Mentions of pathological findings of the lymph nodes. - `Cancer_Dx`: Mentions of cancer diagnoses (such as "breast cancer") or pathological types that are usually used as synonyms for "cancer" (e.g. "carcinoma"). When anatomical references are present, they are included in the Cancer_Dx extraction. - `Cancer_Score`: Clinical or imaging scores that are specific for cancer settings (e.g. "BI-RADS" or "Allred score"). - `Grade`: All pathological grading of tumors (e.g. "grade 1") or degrees of cellular differentiation (e.g. "well-differentiated") - `Histological_Type`: Histological variants or cancer subtypes, such as "papillary", "clear cell" or "medullary". - `Invasion`: Mentions that refer to tumor invasion, such as "invasion" or "involvement". Metastases or lymph node involvement are excluded from this category. - `Metastasis`: Terms that indicate a metastatic disease. Anatomical references are not included in these extractions. - `Pathology_Result`: The findings of a biopsy from the pathology report that is not covered by another entity (e.g. "malignant ductal cells"). - `Performance_Status`: Mentions of performance status scores, such as ECOG and Karnofsky. The name of the score is extracted together with the result (e.g. "ECOG performance status of 4"). - `Staging`: Mentions of cancer stage such as "stage 2b" or "T2N1M0". It also includes words such as "in situ", "early-stage" or "advanced". - `Tumor_Finding`: All nonspecific terms that may be related to tumors, either malignant or benign (for example: "mass", "tumor", "lesion", or "neoplasm"). - `Tumor_Size`: Size of the tumor, including numerical value and unit of measurement (e.g. "3 cm"). ## Predicted Entities `Adenopathy`, `Cancer_Dx`, `Cancer_Score`, `Grade`, `Histological_Type`, `Invasion`, `Metastasis`, `Pathology_Result`, `Performance_Status`, `Staging`, `Tumor_Finding`, `Tumor_Size` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_diagnosis_en_4.0.0_3.0_1666719602276.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_diagnosis_en_4.0.0_3.0_1666719602276.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------|:------------------| | tumor | Tumor_Finding | | adenopathies | Adenopathy | | invasive | Histological_Type | | ductal | Histological_Type | | carcinoma | Cancer_Dx | | metastasis | Metastasis | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_diagnosis| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Histological_Type 354 63 99 453 0.85 0.78 0.81 Staging 234 27 24 258 0.90 0.91 0.90 Cancer_Score 36 15 26 62 0.71 0.58 0.64 Tumor_Finding 1121 83 136 1257 0.93 0.89 0.91 Invasion 154 27 27 181 0.85 0.85 0.85 Tumor_Size 1058 126 71 1129 0.89 0.94 0.91 Adenopathy 66 10 30 96 0.87 0.69 0.77 Performance_Status 116 15 19 135 0.89 0.86 0.87 Pathology_Result 852 686 290 1142 0.55 0.75 0.64 Metastasis 356 15 14 370 0.96 0.96 0.96 Cancer_Dx 1302 88 92 1394 0.94 0.93 0.94 Grade 201 23 35 236 0.90 0.85 0.87 macro_avg 5850 1178 863 6713 0.85 0.83 0.84 micro_avg 5850 1178 863 6713 0.85 0.87 0.86 ``` --- layout: model title: English Named Entity Recognition (from abhibisht89) author: John Snow Labs name: bert_ner_spanbert_large_cased_finetuned_ade_corpus_v2 date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `spanbert-large-cased-finetuned-ade_corpus_v2` is a English model orginally trained by `abhibisht89`. ## Predicted Entities `DRUG`, `ADR` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_spanbert_large_cased_finetuned_ade_corpus_v2_en_3.4.2_3.0_1652096991524.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_spanbert_large_cased_finetuned_ade_corpus_v2_en_3.4.2_3.0_1652096991524.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_spanbert_large_cased_finetuned_ade_corpus_v2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/abhibisht89/spanbert-large-cased-finetuned-ade_corpus_v2 --- layout: model title: Extract textual entities in biomedical texts author: John Snow Labs name: ner_nature_nero_clinical date: 2022-02-08 tags: [ner, en, clinical, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is based on the NERO corpus, capable of extracting general entities. This model is trained to refute the claims made in https://www.nature.com/articles/s41540-021-00200-x regarding Spark NLP's performance and we hereby prove that we can get better than what is claimed. So, **this model is not meant to be used in production.** ## Predicted Entities `Organismpart`, `Chromosome`, `Physicalphenomenon`, `Abstractconcept`, `Gene`, `Meas`, `Machineactivity`, `Warfarin`, `Gen`, `Aminoacidpeptide`, `Language`, `P`, `Quantityormeasurement`, `Disease`, `Process`, `Propernamedgeographicallocation`, `Duration`, `Medicalprocedureordevice`, `Citation`, `Geographicnotproper`, `Atom`, `Gp`, `Medicaldevice`, `Namedentity`, `Unpropernamedgeographicallocation`, `Persongroup`, `Unit`, `Bodypart`, `Unconjugated`, `Timepoint`, `Protein`, `Publishedsourceofinformation`, `Quantity`, `Dr`, `Organism`, `Nonproteinornucleicacidchemical`, `G`, `Researchactivity`, `Drug`, `Measurement`, `Cells`, `Journal`, `Relationshipphrase`, `Medicalprocedure`, `Geographiclocation`, `Groupofpeople`, `Person`, `Tissue`, `Mentalprocess`, `Facility`, `Chemical`, `Geneorproteingroup`, `Ion`, `Food`, `Aminoacid`, `N`, `Biologicalprocess`, `Cell`, `Researchactivty`, `Publicationorcitation`, `Molecularprocess`, `Experimentalfactor`, `Medicalfinding`, `Nucleicacid`, `Laboratoryexperimentalfactor`, `Relationship`, `Geographicallocation`, `Geneorprotein`, `Smallmolecule`, `Partofprotein`, `Thing`, `Quantityormeasure`, `Environmentalfactor`, `Intellectualproduct`, `R`, `Molecule`, `Time`, `Anatomicalpart`, `Cellcomponent`, `Nucleicacidsubstance` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_nature_nero_clinical_en_3.3.4_3.0_1644358495292.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_nature_nero_clinical_en_3.3.4_3.0_1644358495292.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_nature_nero_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature."]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_nature_nero_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.nero_clinical.nature").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | chunk | entity | |---:|:---------------------------------------------|:----------------------| | 0 | perioral cyanosis | Medicalfinding | | 1 | One day | Duration | | 2 | mom | Namedentity | | 3 | tactile temperature | Quantityormeasurement | | 4 | patient Tylenol | Chemical | | 5 | decreased p.o. intake | Medicalprocedure | | 6 | normal breast-feeding | Medicalfinding | | 7 | 20 minutes q.2h | Timepoint | | 8 | 5 to 10 minutes | Duration | | 9 | respiratory congestion | Medicalfinding | | 10 | past 2 days | Duration | | 11 | parents | Persongroup | | 12 | improvement | Process | | 13 | albuterol treatments | Medicalprocedure | | 14 | ER | Bodypart | | 15 | urine output | Quantityormeasurement | | 16 | 8 to 10 wet and 5 dirty diapers per 24 hours | Measurement | | 17 | 4 wet diapers per 24 hours | Measurement | | 18 | Mom | Person | | 19 | diarrhea | Medicalfinding | | 20 | bowel movements | Biologicalprocess | | 21 | soft in nature | Biologicalprocess | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_nature_nero_clinical| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.1 MB| ## References This model is based on https://www.nature.com/articles/s41540-021-00200-x and a response to: https://static-content.springer.com/esm/art%3A10.1038%2Fs41540-021-00200-x/MediaObjects/41540_2021_200_MOESM1_ESM.pdf ## Benchmarking ```bash label tp fp fn prec rec f1 B-Atom 11 7 48 0.6111111 0.18644068 0.2857143 I-Laboratoryexperimentalfactor 0 3 55 0.0 0.0 0.0 B-Disease 489 232 251 0.6782247 0.6608108 0.6694045 B-Partofprotein 77 61 67 0.557971 0.5347222 0.54609925 B-Nonproteinornucleicacidchemical 45 58 190 0.4368932 0.19148937 0.2662722 I-Propernamedgeographicallocation 60 33 31 0.6451613 0.6593407 0.6521739 B-Bodypart 648 336 343 0.6585366 0.65388495 0.6562025 B-Protein 832 504 440 0.6227545 0.6540881 0.63803685 I-Unit 0 0 4 0.0 0.0 0.0 B-Chemical 1390 1066 926 0.5659609 0.6001727 0.5825649 B-Publicationorcitation 2 10 9 0.16666667 0.18181819 0.17391303 I-Smallmolecule 34 201 353 0.14468086 0.087855294 0.10932476 I-Abstractconcept 0 0 4 0.0 0.0 0.0 I-Nucleicacid 449 213 249 0.67824775 0.6432665 0.6602941 B-Drug 306 170 213 0.64285713 0.5895954 0.6150754 B-Thing 0 0 19 0.0 0.0 0.0 I-Citation 6 14 17 0.3 0.26086956 0.27906975 I-Aminoacid 80 43 95 0.6504065 0.45714286 0.53691274 B-Medicalprocedureordevice 1 0 2 1.0 0.33333334 0.5 I-Nucleicacidsubstance 3 3 14 0.5 0.1764706 0.26086956 I-Gp 1244 860 427 0.5912548 0.7444644 0.6590729 I-Geographicallocation 0 18 17 0.0 0.0 0.0 I-Molecule 15 114 58 0.11627907 0.20547946 0.14851485 B-R 0 0 1 0.0 0.0 0.0 I-Measurement 1700 893 546 0.6556113 0.75690114 0.7026245 I-Intellectualproduct 197 251 311 0.43973213 0.38779527 0.41213387 B-Anatomicalpart 0 22 38 0.0 0.0 0.0 B-Gp 3597 1426 818 0.71610594 0.81472254 0.7622378 B-Person 105 42 80 0.71428573 0.5675676 0.63253015 I-Aminoacidpeptide 55 38 35 0.5913978 0.6111111 0.6010929 B-Environmentalfactor 24 30 40 0.44444445 0.375 0.40677968 B-Cellcomponent 188 146 191 0.56287426 0.49604222 0.5273493 I-Groupofpeople 1 1 10 0.5 0.09090909 0.15384614 I-Chromosome 39 27 12 0.59090906 0.7647059 0.6666667 B-G 0 0 1 0.0 0.0 0.0 I-Publishedsourceofinformation 122 100 131 0.5495495 0.48221344 0.5136842 I-Disease 710 313 239 0.69403714 0.74815595 0.72008115 I-Time 19 32 86 0.37254903 0.18095239 0.24358974 I-Relationship 41 18 33 0.69491524 0.5540541 0.6165413 I-Nonproteinornucleicacidchemical 32 87 257 0.26890758 0.11072665 0.15686275 I-Molecularprocess 1257 1057 589 0.5432152 0.68093175 0.6043269 I-Persongroup 587 199 233 0.7468193 0.71585363 0.7310087 B-Laboratoryexperimentalfactor 0 2 42 0.0 0.0 0.0 I-Mentalprocess 24 30 97 0.44444445 0.1983471 0.2742857 B-Aminoacidpeptide 33 26 44 0.55932206 0.42857143 0.48529413 B-Food 63 29 54 0.6847826 0.53846157 0.6028708 B-Journal 0 0 3 0.0 0.0 0.0 I-Quantityormeasure 0 2 4 0.0 0.0 0.0 I-Cell 1035 212 252 0.829992 0.8041958 0.81689036 B-Tissue 57 41 53 0.5816327 0.5181818 0.5480769 I-Medicaldevice 51 58 53 0.4678899 0.4903846 0.47887325 B-Mentalprocess 57 49 156 0.5377358 0.26760563 0.35736677 I-Bodypart 659 406 366 0.61877936 0.6429268 0.630622 I-Researchactivity 1073 568 410 0.65386957 0.7235334 0.6869398 I-Atom 11 2 51 0.84615386 0.17741935 0.29333335 B-Namedentity 173 423 504 0.29026845 0.25553915 0.2717989 B-Quantityormeasure 2 2 6 0.5 0.25 0.33333334 B-Citation 0 2 3 0.0 0.0 0.0 I-Cellcomponent 183 166 180 0.5243553 0.5041322 0.51404494 B-Unit 0 0 3 0.0 0.0 0.0 I-Person 41 50 33 0.45054945 0.5540541 0.49696973 I-Quantityormeasurement 202 418 557 0.32580644 0.26613966 0.2929659 B-Organismpart 23 54 41 0.2987013 0.359375 0.32624114 B-Cell 723 206 231 0.7782562 0.7578616 0.76792353 I-Chemical 1898 1128 832 0.62723064 0.6952381 0.65948576 I-Medicalfinding 1749 1267 1362 0.5799072 0.56219864 0.5709156 B-Process 1522 1421 1714 0.51715934 0.47033376 0.49263635 I-Food 75 39 60 0.65789473 0.5555556 0.60240966 I-Duration 344 269 169 0.5611746 0.6705653 0.61101246 I-Experimentalfactor 59 173 200 0.25431034 0.22779922 0.24032587 I-Quantity 742 670 750 0.52549577 0.49731904 0.5110193 B-Physicalphenomenon 1 2 11 0.33333334 0.083333336 0.13333334 I-Medicalprocedureordevice 3 0 3 1.0 0.5 0.6666667 B-Aminoacid 86 41 109 0.6771653 0.44102564 0.5341615 B-Quantity 613 554 645 0.5252785 0.4872814 0.5055671 B-Cells 0 0 2 0.0 0.0 0.0 I-Gene 134 44 95 0.752809 0.58515286 0.65847665 B-Medicalfinding 1913 1389 1296 0.5793458 0.59613585 0.5876209 I-Tissue 70 58 42 0.546875 0.625 0.5833333 B-Molecule 20 48 62 0.29411766 0.24390244 0.26666665 I-Organism 959 373 282 0.71997 0.7727639 0.74543333 I-Medicalprocedure 775 475 370 0.62 0.6768559 0.64718163 B-Unpropernamedgeographicallocation 7 5 25 0.5833333 0.21875 0.3181818 I-Timepoint 3 9 32 0.25 0.08571429 0.12765957 I-Organismpart 15 47 27 0.24193548 0.35714287 0.28846154 I-Biologicalprocess 798 787 835 0.50347 0.48867115 0.49596024 B-Time 47 49 97 0.48958334 0.3263889 0.39166668 B-Experimentalfactor 40 100 136 0.2857143 0.22727273 0.2531646 B-Nucleicacid 364 234 337 0.6086956 0.5192582 0.5604311 B-Propernamedgeographicallocation 112 38 22 0.74666667 0.8358209 0.7887324 B-Publishedsourceofinformation 252 95 125 0.7262248 0.66843504 0.69613266 I-Unpropernamedgeographicallocation 5 4 21 0.5555556 0.1923077 0.2857143 I-Protein 1114 717 449 0.6084107 0.7127319 0.65645254 B-Molecularprocess 907 696 609 0.5658141 0.59828496 0.5815967 B-Quantityormeasurement 171 293 438 0.36853448 0.28078818 0.31873253 B-Intellectualproduct 159 175 215 0.4760479 0.42513368 0.44915253 B-Persongroup 783 245 223 0.76167315 0.77833 0.7699115 I-Cells 0 0 2 0.0 0.0 0.0 B-Researchactivity 869 448 373 0.65983295 0.69967794 0.67917156 I-Environmentalfactor 20 18 35 0.5263158 0.36363637 0.43010756 B-Gene 64 37 105 0.63366336 0.37869823 0.47407413 B-Groupofpeople 3 2 10 0.6 0.23076923 0.33333334 B-Geneorproteingroup 241 191 203 0.5578704 0.5427928 0.5502283 B-Facility 119 134 180 0.47035572 0.3979933 0.43115944 B-Timepoint 10 15 30 0.4 0.25 0.30769232 B-Organism 869 290 297 0.7497843 0.745283 0.7475268 B-Duration 274 186 137 0.59565216 0.6666667 0.6291619 I-Facility 182 212 226 0.46192893 0.44607842 0.45386532 I-Process 589 880 1114 0.40095302 0.34586024 0.3713745 B-Language 0 0 2 0.0 0.0 0.0 B-Medicaldevice 29 44 56 0.39726028 0.34117648 0.36708862 B-Ion 54 23 37 0.7012987 0.5934066 0.64285713 I-Partofprotein 126 131 153 0.49027237 0.4516129 0.47014928 B-Gen 0 0 1 0.0 0.0 0.0 B-Geneorprotein 0 1 64 0.0 0.0 0.0 I-Thing 0 0 16 0.0 0.0 0.0 I-Gen 0 0 3 0.0 0.0 0.0 I-Geneorproteingroup 398 345 286 0.5356662 0.58187133 0.5578136 B-Abstractconcept 0 0 7 0.0 0.0 0.0 B-Chromosome 37 24 24 0.60655737 0.60655737 0.60655737 B-Relationship 133 64 59 0.6751269 0.6927083 0.6838046 B-Smallmolecule 19 62 229 0.2345679 0.076612905 0.115501516 I-Physicalphenomenon 0 2 7 0.0 0.0 0.0 I-Ion 102 41 126 0.7132867 0.4473684 0.5498652 I-Drug 173 133 169 0.5653595 0.50584793 0.5339506 I-Anatomicalpart 0 63 48 0.0 0.0 0.0 B-Measurement 639 451 298 0.5862385 0.68196374 0.6304884 I-Publicationorcitation 10 46 52 0.17857143 0.16129032 0.16949153 B-Geographicallocation 4 16 20 0.2 0.16666667 0.18181819 I-Journal 0 0 11 0.0 0.0 0.0 B-Relationshipphrase 0 0 1 0.0 0.0 0.0 B-Nucleicacidsubstance 2 2 16 0.5 0.11111111 0.18181819 B-Biologicalprocess 779 703 831 0.525641 0.48385093 0.503881 I-Geneorprotein 0 2 33 0.0 0.0 0.0 B-Medicalprocedure 820 450 346 0.6456693 0.703259 0.6732348 I-Namedentity 140 447 381 0.23850085 0.268714 0.25270757 Macro-average 41221 28282 28209 0.4370514 0.3767836 0.40468597 Micro-average 41221 28282 28209 0.5930823 0.5937059 0.5933939 ``` --- layout: model title: MNDA / NDA Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_nda_agreements date: 2023-02-08 tags: [nda, mnda, en, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_nda_agreements` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `nda` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `nda`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_nda_agreements_en_1.0.0_3.0_1675880803682.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_nda_agreements_en_1.0.0_3.0_1675880803682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_nda_agreements", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[nda]| |[other]| |[other]| |[nda]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_nda_agreements| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support nda 0.93 0.96 0.95 135 other 0.97 0.95 0.96 181 accuracy - - 0.95 316 macro-avg 0.95 0.95 0.95 316 weighted-avg 0.95 0.95 0.95 316 ``` --- layout: model title: Javanese BertForMaskedLM Small Cased model (from w11wo) author: John Snow Labs name: bert_embeddings_javanese_small date: 2022-12-02 tags: [jv, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: jv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `javanese-bert-small` is a Javanese model originally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_small_jv_4.2.4_3.0_1670022474239.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_javanese_small_jv_4.2.4_3.0_1670022474239.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_javanese_small","jv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_javanese_small","jv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_javanese_small| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|jv| |Size:|410.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/javanese-bert-small - https://arxiv.org/abs/1810.04805 - https://github.com/sgugger - https://github.com/piegu/fastai-projects/blob/master/finetuning-English-GPT2-any-language-Portuguese-HuggingFace-fastaiv2.ipynb - https://w11wo.github.io/ --- layout: model title: Fast Neural Machine Translation Model from Tongan to English author: John Snow Labs name: opus_mt_to_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, to, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `to` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_to_en_xx_2.7.0_2.4_1609167464976.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_to_en_xx_2.7.0_2.4_1609167464976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_to_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Closings Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_closings_bert date: 2023-03-05 tags: [en, legal, classification, clauses, closings, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Closings` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Closings`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_closings_bert_en_1.0.0_3.0_1678049939782.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_closings_bert_en_1.0.0_3.0_1678049939782.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_closings_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Closings]| |[Other]| |[Other]| |[Closings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_closings_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Closings 0.96 0.92 0.94 60 Other 0.94 0.98 0.96 83 accuracy - - 0.95 143 macro-avg 0.95 0.95 0.95 143 weighted-avg 0.95 0.95 0.95 143 ``` --- layout: model title: Pipeline to Detect Radiology Concepts - WIP (biobert) author: John Snow Labs name: jsl_rd_ner_wip_greedy_biobert_pipeline date: 2023-03-20 tags: [licensed, clinical, en, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [jsl_rd_ner_wip_greedy_biobert](https://nlp.johnsnowlabs.com/2021/07/26/jsl_rd_ner_wip_greedy_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_biobert_pipeline_en_4.3.0_3.2_1679310411354.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_biobert_pipeline_en_4.3.0_3.2_1679310411354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("jsl_rd_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") text = '''Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("jsl_rd_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") val text = "Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.wip_greedy_biobert.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------|--------:|------:|:--------------------------|-------------:| | 0 | Bilateral | 0 | 8 | Direction | 0.9875 | | 1 | breast | 10 | 15 | BodyPart | 0.6109 | | 2 | ultrasound | 17 | 26 | ImagingTest | 0.7902 | | 3 | ovoid mass | 78 | 87 | ImagingFindings | 0.42185 | | 4 | 0.5 x 0.5 x 0.4 | 113 | 127 | Measurements | 0.9406 | | 5 | cm | 129 | 130 | Units | 1 | | 6 | left | 190 | 193 | Direction | 0.5566 | | 7 | shoulder | 195 | 202 | BodyPart | 0.6228 | | 8 | mass | 210 | 213 | ImagingFindings | 0.9463 | | 9 | isoechoic echotexture | 228 | 248 | ImagingFindings | 0.4332 | | 10 | muscle | 266 | 271 | BodyPart | 0.7148 | | 11 | internal color flow | 294 | 312 | ImagingFindings | 0.3726 | | 12 | benign fibrous tissue | 334 | 354 | ImagingFindings | 0.484533 | | 13 | lipoma | 361 | 366 | Disease_Syndrome_Disorder | 0.8955 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_rd_ner_wip_greedy_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265904` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904_en_4.0.0_3.0_1655985110516.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904_en_4.0.0_3.0_1655985110516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265904").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265904| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265904 --- layout: model title: German asr_wav2vec2_xls_r_300m_english_by_aware_ai TFWav2Vec2ForCTC from aware-ai author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_english_by_aware_ai` is a German model originally trained by aware-ai. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai_de_4.2.0_3.0_1664097471522.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai_de_4.2.0_3.0_1664097471522.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_english_by_aware_ai| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Nyaneka to English Pipeline author: John Snow Labs name: translate_nyk_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, nyk, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `nyk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_nyk_en_xx_2.7.0_2.4_1609689243589.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_nyk_en_xx_2.7.0_2.4_1609689243589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_nyk_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_nyk_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.nyk.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_nyk_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Exceptions Clause Binary Classifier author: John Snow Labs name: legclf_exceptions_clause date: 2023-02-13 tags: [en, legal, classification, exceptions, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `exceptions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `exceptions`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_exceptions_clause_en_1.0.0_3.0_1676303705879.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_exceptions_clause_en_1.0.0_3.0_1676303705879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_exceptions_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[exceptions]| |[other]| |[other]| |[exceptions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_exceptions_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support exceptions 1.00 1.00 1.00 17 other 1.00 1.00 1.00 8 accuracy - - 1.00 25 macro-avg 1.00 1.00 1.00 25 weighted-avg 1.00 1.00 1.00 25 ``` --- layout: model title: Fast Neural Machine Translation Model from Setswana to English author: John Snow Labs name: opus_mt_tn_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tn, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tn_en_xx_2.7.0_2.4_1609163658011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tn_en_xx_2.7.0_2.4_1609163658011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tn_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-2_H-512_A-8_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185242186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna_en_4.0.0_3.0_1654185242186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.uncased_2l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_2_H_512_A_8_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|83.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-2_H-512_A-8_squad2_covid-qna --- layout: model title: Fast Neural Machine Translation Model from Arabic to Greek author: John Snow Labs name: opus_mt_ar_el date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, el, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: el {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_el_xx_3.1.0_2.4_1622550439728.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_el_xx_3.1.0_2.4_1622550439728.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_el| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect PHI for Deidentification purposes (Spanish, reduced entities, augmented data, Roberta) author: John Snow Labs name: ner_deid_generic_roberta_augmented date: 2022-02-15 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 8 entities (1 more than the `ner_deid_generic_roberta` ner model). This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf dataset, several data augmentation mechanisms and has been augmented with MEDDOCAN Spanish Deidentification corpus (compared to `ner_deid_generic_roberta` which does not include it). It's a generalized version of `ner_deid_subentity_roberta_augmented`. This is a Roberta embeddings based model. You also have available the `ner_deid_generic_augmented` that uses Sciwi 300d embeddings. ## Predicted Entities `CONTACT`, `NAME`, `DATE`, `ID`, `LOCATION`, `PROFESSION`, `AGE`, `SEX` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_roberta_augmented_es_3.3.4_2.4_1644926722996.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_roberta_augmented_es_3.3.4_2.4_1644926722996.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") roberta_embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_generic_roberta_augmented", "es", "clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner]) text = [''' Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] df = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(df).transform(df) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") roberta_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_generic_augmented", "es", "clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, roberta_embeddings, clinical_ner)) val text = "Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos." val df = Seq(text).toDF("text") val results = pipeline.fit(df).transform(df) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.generic.roberta").predict(""" Antonio Miguel Martínez, un varón de 35 años de edad, de profesión auxiliar de enfermería y nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14 de Marzo y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+------------+ | token| ner_label| +------------+------------+ | Antonio| B-NAME| | Miguel| I-NAME| | Martínez| I-NAME| | ,| O| | un| B-SEX| | varón| I-SEX| | de| O| | 35| B-AGE| | años| O| | de| O| | edad| O| | ,| O| | de| O| | profesión| O| | auxiliar|B-PROFESSION| | de|I-PROFESSION| | enfermería|I-PROFESSION| | y| O| | nacido| O| | en| O| | Cadiz| B-LOCATION| | ,| O| | España| B-LOCATION| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19|B-PROFESSION| | el| O| | dia| O| | 14| O| | de| O| | Marzo| O| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| B-LOCATION| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica| B-LOCATION| | San| I-LOCATION| | Carlos| I-LOCATION| | .| O| +------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_roberta_augmented| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|16.3 MB| ## References - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) - [MeddoCan](https://temu.bsc.es/meddocan/) ## Benchmarking ```bash label tp fp fn total precision recall f1 CONTACT 177.0 3.0 6.0 183.0 0.9833 0.9672 0.9752 NAME 1963.0 159.0 123.0 2086.0 0.9251 0.941 0.933 DATE 953.0 18.0 16.0 969.0 0.9815 0.9835 0.9825 ORGANIZATION 2320.0 520.0 362.0 2682.0 0.8169 0.865 0.8403 ID 63.0 7.0 1.0 64.0 0.9 0.9844 0.9403 SEX 619.0 14.0 8.0 627.0 0.9779 0.9872 0.9825 LOCATION 2388.0 470.0 423.0 2811.0 0.8355 0.8495 0.8425 PROFESSION 233.0 15.0 28.0 261.0 0.9395 0.8927 0.9155 AGE 516.0 16.0 3.0 519.0 0.9699 0.9942 0.9819 macro - - - - - - 0.9326 micro - - - - - - 0.8943 ``` --- layout: model title: English BertForSequenceClassification Tiny Cased model (from mrm8488) author: John Snow Labs name: bert_sequence_classifier_tiny_finetuned_sms_spam_detection date: 2022-07-13 tags: [en, open_source, bert, sequence_classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-finetuned-sms-spam-detection` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_tiny_finetuned_sms_spam_detection_en_4.0.0_3.0_1657720825763.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_tiny_finetuned_sms_spam_detection_en_4.0.0_3.0_1657720825763.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = BertForSequenceClassification.pretrained("bert_sequence_classifier_tiny_finetuned_sms_spam_detection","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = BertForSequenceClassification.pretrained("bert_sequence_classifier_tiny_finetuned_sms_spam_detection","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_tiny_finetuned_sms_spam_detection| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|16.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/bert-tiny-finetuned-sms-spam-detection --- layout: model title: Swedish BertForMaskedLM Base Cased model (from KBLab) author: John Snow Labs name: bert_embeddings_kblab_base_swedish_cased date: 2022-12-02 tags: [sv, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: sv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-swedish-cased` is a Swedish model originally trained by `KBLab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_kblab_base_swedish_cased_sv_4.2.4_3.0_1670019031106.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_kblab_base_swedish_cased_sv_4.2.4_3.0_1670019031106.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kblab_base_swedish_cased","sv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kblab_base_swedish_cased","sv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_kblab_base_swedish_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|sv| |Size:|468.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/KBLab/bert-base-swedish-cased --- layout: model title: English RobertaForQuestionAnswering (from z-uo) author: John Snow Labs name: roberta_qa_roberta_qasper date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-qasper` is a English model originally trained by `z-uo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_qasper_en_4.0.0_3.0_1655738258703.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_qasper_en_4.0.0_3.0_1655738258703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_qasper","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_qasper","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.by_z-uo").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_qasper| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/z-uo/roberta-qasper --- layout: model title: Legal Title Clause Binary Classifier author: John Snow Labs name: legclf_title_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `title` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `title` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_title_clause_en_1.0.0_3.2_1660123118000.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_title_clause_en_1.0.0_3.2_1660123118000.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_title_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[title]| |[other]| |[other]| |[title]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_title_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.98 0.96 179 title 0.93 0.86 0.90 65 accuracy - - 0.95 244 macro-avg 0.94 0.92 0.93 244 weighted-avg 0.95 0.95 0.95 244 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_8_h_512 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-8_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_512_zh_4.2.4_3.0_1670326051660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_512_zh_4.2.4_3.0_1670326051660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_8_h_512| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|137.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-8_H-512 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Legal Lease agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_lease_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_lease_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class lease-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `lease-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_lease_agreement_en_1.0.0_3.0_1666620984031.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_lease_agreement_en_1.0.0_3.0_1666620984031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_lease_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[lease-agreement]| |[other]| |[other]| |[lease-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_lease_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support lease-agreement 0.94 0.97 0.96 33 other 0.98 0.97 0.98 62 accuracy - - 0.97 95 macro-avg 0.96 0.97 0.97 95 weighted-avg 0.97 0.97 0.97 95 ``` --- layout: model title: Pipeline to Mapping ICD10-CM Codes with Their Corresponding ICD-9-CM Codes author: John Snow Labs name: icd10_icd9_mapping date: 2023-03-29 tags: [en, licensed, icd10cm, icd9, pipeline, chunk_mapping] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `icd10_icd9_mapper` model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10_icd9_mapping_en_4.3.2_3.2_1680118330093.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10_icd9_mapping_en_4.3.2_3.2_1680118330093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd10_icd9_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(Z833 A0100 A000) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10_icd9_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(Z833 A0100 A000) ``` {:.nlu-block} ```python import nlu nlu.load("en.icd10_icd9.mapping").predict("""Put your text here.""") ```

## Results ```bash | | icd10_code | icd9_code | |---:|:--------------------|:-------------------| | 0 | Z833 | A0100 | A000 | V180 | 0020 | 0010 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10_icd9_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|589.6 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Temporality / Certainty Assertion Status author: John Snow Labs name: finassertion_time date: 2022-09-27 tags: [en, licensed] task: Assertion Status language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an Assertion Status Model aimed to detect temporality (PRESENT, PAST, FUTURE) or Certainty (POSSIBLE) in your financial documents ## Predicted Entities `PRESENT`, `PAST`, `FUTURE`, `POSSIBLE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINASSERTION_TEMPORALITY){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finassertion_time_en_1.0.0_3.0_1664274273525.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finassertion_time_en_1.0.0_3.0_1664274273525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash chunk,begin,end,entity_type,assertion CTO,47,49,ROLE,FUTURE ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finassertion_time| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, doc_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|2.2 MB| ## References In-house annotations on financial and legal corpora ## Benchmarking ```bash label tp fp fn prec rec f1 PRESENT 201 11 16 0.9481132 0.92626727 0.937063 POSSIBLE 171 3 6 0.98275864 0.9661017 0.9743589 FUTURE 119 6 4 0.952 0.96747965 0.95967746 PAST 270 16 10 0.9440559 0.96428573 0.9540636 Macro-average 761 36 36 0.9567319 0.9560336 0.95638263 Micro-average 761 36 36 0.9548306 0.9548306 0.9548306 ``` --- layout: model title: Pipeline to Extract Anatomical Entities from Oncology Texts author: John Snow Labs name: ner_oncology_anatomy_general_healthcare_pipeline date: 2023-03-08 tags: [licensed, clinical, oncology, en, ner, anatomy] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_anatomy_general_healthcare](https://nlp.johnsnowlabs.com/2023/01/11/ner_oncology_anatomy_general_healthcare_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_healthcare_pipeline_en_4.3.0_3.2_1678268209780.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_general_healthcare_pipeline_en_4.3.0_3.2_1678268209780.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_anatomy_general_healthcare_pipeline", "en", "clinical/models") text = " The patient presented a mass in her left breast, and a possible metastasis in her lungs and in her liver. " result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_anatomy_general_healthcare_pipeline", "en", "clinical/models") val text = " The patient presented a mass in her left breast, and a possible metastasis in her lungs and in her liver. " val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:---------|--------:|------:|:----------------|-------------:| | 0 | left | 37 | 40 | Direction | 0.9948 | | 1 | breast | 42 | 47 | Anatomical_Site | 0.5814 | | 2 | lungs | 83 | 87 | Anatomical_Site | 0.9486 | | 3 | liver | 100 | 104 | Anatomical_Site | 0.9646 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_anatomy_general_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|533.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_scrambled_squad_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-10` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_10_en_4.0.0_3.0_1655733955391.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_10_en_4.0.0_3.0_1655733955391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_scrambled_10.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_scrambled_squad_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-10 --- layout: model title: Translate Swedish to English Pipeline author: John Snow Labs name: translate_sv_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, sv, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `sv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_sv_en_xx_2.7.0_2.4_1609699177957.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_sv_en_xx_2.7.0_2.4_1609699177957.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_sv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_sv_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.sv.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_sv_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Czech asr_wav2vec2_large_xlsr_czech TFWav2Vec2ForCTC from arampacha author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_czech date: 2022-09-25 tags: [wav2vec2, cs, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: cs edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_czech` is a Czech model originally trained by arampacha. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_czech_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_czech_cs_4.2.0_3.0_1664120879670.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_czech_cs_4.2.0_3.0_1664120879670.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_czech', lang = 'cs') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_czech", lang = "cs") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_czech| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|cs| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Multilingual T5ForConditionalGeneration Small Cased model (from hackathon-pln-es) author: John Snow Labs name: t5_small_finetuned_spanish_to_quechua date: 2023-01-31 tags: [qu, es, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-finetuned-spanish-to-quechua` is a Multilingual model originally trained by `hackathon-pln-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_spanish_to_quechua_xx_4.3.0_3.0_1675126096661.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_spanish_to_quechua_xx_4.3.0_3.0_1675126096661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_finetuned_spanish_to_quechua","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_finetuned_spanish_to_quechua","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_finetuned_spanish_to_quechua| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|282.6 MB| ## References - https://huggingface.co/hackathon-pln-es/t5-small-finetuned-spanish-to-quechua --- layout: model title: Part of Speech for Telugu (pos_mtg) author: John Snow Labs name: pos_mtg date: 2021-03-10 tags: [open_source, pos, te] supported: true task: Part of Speech Tagging language: te edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_mtg_te_2.7.5_2.4_1615400812325.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_mtg_te_2.7.5_2.4_1615400812325.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_mtg", "te") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['ఆయన వస్తున్నారా , లేదా ?']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_mtg", "te") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("ఆయన వస్తున్నారా , లేదా ?").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""ఆయన వస్తున్నారా , లేదా ?""] token_df = nlu.load('te.pos.mtg').predict(text) token_df ```

## Results ```bash +------------------------+--------------------------------+ |text |result | +------------------------+--------------------------------+ |ఆయన వస్తున్నారా , లేదా ?|[PRON, VERB, PUNCT, VERB, PUNCT]| +------------------------+--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_mtg| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|te| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.50 | 0.40 | 0.44 | 5 | | ADP | 0.75 | 0.43 | 0.55 | 7 | | ADV | 0.78 | 0.68 | 0.72 | 31 | | CCONJ | 0.00 | 0.00 | 0.00 | 1 | | DET | 0.89 | 0.89 | 0.89 | 18 | | INTJ | 0.00 | 0.00 | 0.00 | 0 | | NOUN | 0.82 | 0.76 | 0.79 | 171 | | NUM | 0.83 | 0.42 | 0.56 | 12 | | PART | 0.00 | 0.00 | 0.00 | 2 | | PRON | 0.88 | 0.93 | 0.91 | 122 | | PROPN | 0.69 | 0.86 | 0.77 | 21 | | PUNCT | 0.99 | 0.99 | 0.99 | 165 | | SCONJ | 0.71 | 1.00 | 0.83 | 5 | | VERB | 0.84 | 0.92 | 0.88 | 161 | | accuracy | | | 0.87 | 721 | | macro avg | 0.62 | 0.59 | 0.59 | 721 | | weighted avg | 0.87 | 0.87 | 0.86 | 721 | ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson TFWav2Vec2ForCTC from izzy-lazerson author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson` is a English model originally trained by izzy-lazerson. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson_en_4.2.0_3.0_1664112196900.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson_en_4.2.0_3.0_1664112196900.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_izzy_lazerson| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten` is a English model originally trained by patrickvonplaten. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_en_4.2.0_3.0_1664105340184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_en_4.2.0_3.0_1664105340184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Italian BertForMaskedLM Base Cased model (from dbmdz) author: John Snow Labs name: bert_embeddings_base_italian_cased date: 2022-12-02 tags: [it, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-italian-cased` is a Italian model originally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_cased_it_4.2.4_3.0_1670017918448.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_cased_it_4.2.4_3.0_1670017918448.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_cased","it") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_cased","it") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_italian_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|412.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-cased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Human Rights Articles Classification author: John Snow Labs name: legclf_human_rights date: 2022-08-09 tags: [es, legal, conventions, classification, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Roberta-based Legal Sequence Classifier NLP model to label texts about Human Rights in Spanish as one of the following categories: - `Artículo 1. Obligación de Respetar los Derechos` - `Artículo 2. Deber de Adoptar Disposiciones de Derecho Interno` - `Artículo 3. Derecho al Reconocimiento de la Personalidad Jurídica` - `Artículo 4. Derecho a la Vida` - `Artículo 5. Derecho a la Integridad Personal` - `Artículo 6. Prohibición de la Esclavitud y Servidumbre` - `Artículo 7. Derecho a la Libertad Personal` - `Artículo 8. Garantías Judiciales` - `Artículo 9. Principio de Legalidad y de Retroactividad` - `Artículo 11. Protección de la Honra y de la Dignidad` - `Artículo 12. Libertad de Conciencia y de Religión` - `Artículo 13. Libertad de Pensamiento y de Expresión` - `Artículo 14. Derecho de Rectificación o Respuesta` - `Artículo 15. Derecho de Reunión` - `Artículo 16. Libertad de Asociación` - `Artículo 17. Protección a la Familia` - `Artículo 18. Derecho al Nombre` - `Artículo 19. Derechos del Niño` - `Artículo 20. Derecho a la Nacionalidad` - `Artículo 22. Derecho de Circulación y de Residencia` - `Artículo 23. Derechos Políticos` - `Artículo 24. Igualdad ante la Ley` - `Artículo 25. Protección Judicial` - `Artículo 26. Desarrollo Progresivo` - `Artículo 27. Suspensión de Garantías` - `Artículo 28. Cláusula Federal`, `Artículo 21. Derecho a la Propiedad Privada` - `Artículo_29_Normas_de_Interpretación` - `Artículo 30. Alcance de las Restricciones` - `Artículo 63.1 Reparaciones` This model was originally trained with 6089 legal texts (see the original work [here](https://huggingface.co/hackathon-pln-es/jurisbert-clas-art-convencion-americana-dh) about the American Convention of Human Rights. It has been finetuned with the International Convention of Human Rights and other similar documents (as, for example, https://www.ohchr.org/sites/default/files/UDHR/Documents/UDHR_Translations/spn.pdf). ## Predicted Entities `Artículo 1. Obligación de Respetar los Derechos`, `Artículo 2. Deber de Adoptar Disposiciones de Derecho Interno`, `Artículo 3. Derecho al Reconocimiento de la Personalidad Jurídica`, `Artículo 4. Derecho a la Vida`, `Artículo 5. Derecho a la Integridad Personal`, `Artículo 6. Prohibición de la Esclavitud y Servidumbre`, `Artículo 7. Derecho a la Libertad Personal`, `Artículo 8. Garantías Judiciales`, `Artículo 9. Principio de Legalidad y de Retroactividad`, `Artículo 11. Protección de la Honra y de la Dignidad`, `Artículo 12. Libertad de Conciencia y de Religión`, `Artículo 13. Libertad de Pensamiento y de Expresión`, `Artículo 14. Derecho de Rectificación o Respuesta`, `Artículo 15. Derecho de Reunión`, `Artículo 16. Libertad de Asociación`, `Artículo 17. Protección a la Familia`, `Artículo 18. Derecho al Nombre`, `Artículo 19. Derechos del Niño`, `Artículo 20. Derecho a la Nacionalidad`, `Artículo 22. Derecho de Circulación y de Residencia`, `Artículo 23. Derechos Políticos`, `Artículo 24. Igualdad ante la Ley`, `Artículo 25. Protección Judicial`, `Artículo 26. Desarrollo Progresivo`, `Artículo 27. Suspensión de Garantías`, `Artículo 28. Cláusula Federal`, `Artículo 21. Derecho a la Propiedad Privada`, `Artículo 29. Normas de Interpretación`, `Artículo 30. Alcance de las Restricciones`, `Artículo 63.1 Reparaciones` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_human_rights_en_1.0.0_3.2_1660057114857.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_human_rights_en_1.0.0_3.2_1660057114857.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+-------------------------------------------------+ | text| result| +--------------------+-------------------------------------------------+ |Todos los seres h...|[Artículo 1. Obligación de Respetar los Derechos]| +--------------------+-------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_human_rights| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model was originally trained on 6089 legal texts (see the original work [here](https://huggingface.co/hackathon-pln-es/jurisbert-clas-art-convencion-americana-dh) about the American Convention of Human Rights. It has been finetuned with the International Convention of Human Rights and other similar documents (as, for example, https://www.ohchr.org/sites/default/files/UDHR/Documents/UDHR_Translations/spn.pdf). ## Benchmarking ```bash label precision recall f1-score support accuracy - - 0.91 98 macro-avg 0.92 0.91 0.91 98 weighted-avg 0.92 0.90 0.91 98 ``` --- layout: model title: Arabic BertForQuestionAnswering Cased model (from Sh3ra) author: John Snow Labs name: bert_qa_arabert_finetuned_arcd date: 2022-07-07 tags: [ar, open_source, bert, question_answering] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `arabert-finetuned-arcd` is a Arabic model originally trained by `Sh3ra`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_arabert_finetuned_arcd_ar_4.0.0_3.0_1657182523872.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_arabert_finetuned_arcd_ar_4.0.0_3.0_1657182523872.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_arabert_finetuned_arcd","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_arabert_finetuned_arcd","ar") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("ما هو اسمي؟", "اسمي كلارا وأنا أعيش في بيركلي.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_arabert_finetuned_arcd| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ar| |Size:|505.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Sh3ra/arabert-finetuned-arcd --- layout: model title: Legal Monetary Relations Document Classifier (EURLEX) author: John Snow Labs name: legclf_monetary_relations_bert date: 2023-03-06 tags: [en, legal, classification, clauses, monetary_relations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_monetary_relations_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Monetary_Relations or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Monetary_Relations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_monetary_relations_bert_en_1.0.0_3.0_1678111802226.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_monetary_relations_bert_en_1.0.0_3.0_1678111802226.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_monetary_relations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Monetary_Relations]| |[Other]| |[Other]| |[Monetary_Relations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_monetary_relations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Monetary_Relations 0.75 0.88 0.81 24 Other 0.90 0.79 0.84 34 accuracy - - 0.83 58 macro-avg 0.82 0.83 0.83 58 weighted-avg 0.84 0.83 0.83 58 ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2_en_4.0.0_3.0_1657192626029.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2_en_4.0.0_3.0_1657192626029.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|387.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-2 --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0_en_4.0.0_3.0_1655732836624.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0_en_4.0.0_3.0_1655732836624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_512d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|432.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-0 --- layout: model title: Word Embeddings for French (word2vec_wac_200) author: John Snow Labs name: word2vec_wac_200 date: 2022-02-01 tags: [fr, open_source] task: Embeddings language: fr edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This French Word2Vec model was trained by [Jean-Philippe Fauconnier](https://fauconnier.github.io/) on the [frWaC Corpus](https://wacky.sslmit.unibo.it/doku.php?id=corpora) over a window size of 100 and dimensions of 200. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/word2vec_wac_200_fr_3.4.0_3.0_1643751580352.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/word2vec_wac_200_fr_3.4.0_3.0_1643751580352.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|word2vec_wac_200| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|118.0 MB| |Case sensitive:|false| |Dimension:|200| ## References This model was trained by [Jean-Philippe Fauconnier](https://fauconnier.github.io/) on the [frWaC Corpus](https://wacky.sslmit.unibo.it/doku.php?id=corpora). [[1]](#1) [1] Fauconnier, Jean-Philippe (2015), French Word Embeddings, http://fauconnier.github.io --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Raphaelg9) author: John Snow Labs name: distilbert_qa_raphaelg9_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Raphaelg9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_raphaelg9_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769023119.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_raphaelg9_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769023119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_raphaelg9_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_raphaelg9_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_raphaelg9_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Raphaelg9/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Execution Clause Binary Classifier author: John Snow Labs name: legclf_execution_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `execution` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `execution` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_execution_clause_en_1.0.0_3.2_1660123518825.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_execution_clause_en_1.0.0_3.2_1660123518825.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_execution_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[execution]| |[other]| |[other]| |[execution]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_execution_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support execution 0.87 0.77 0.82 44 other 0.89 0.94 0.91 84 accuracy - - 0.88 128 macro-avg 0.88 0.86 0.87 128 weighted-avg 0.88 0.88 0.88 128 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from victorlee071200) author: John Snow Labs name: distilbert_qa_victorlee071200_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `victorlee071200`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_victorlee071200_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773002397.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_victorlee071200_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773002397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_victorlee071200_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_victorlee071200_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_victorlee071200_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/victorlee071200/distilbert-base-uncased-finetuned-squad --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_ff3000 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-ff3000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff3000_en_4.3.0_3.0_1675120955500.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff3000_en_4.3.0_3.0_1675120955500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_ff3000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_ff3000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_ff3000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|171.8 MB| ## References - https://huggingface.co/google/t5-efficient-small-ff3000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Abkhazian asr_wav2vec2_xls_r_300m_ab_CV8 TFWav2Vec2ForCTC from emre author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m_ab_CV8 date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_ab_CV8` is a Abkhazian model originally trained by emre. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_ab_CV8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_ab_CV8_ab_4.2.0_3.0_1664020110360.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_ab_CV8_ab_4.2.0_3.0_1664020110360.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m_ab_CV8', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m_ab_CV8", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m_ab_CV8| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Arabic XlmRoBertaForQuestionAnswering (from salti) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_large_arabic_qa date: 2022-06-23 tags: [ar, open_source, question_answering, xlmroberta] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-large-arabic_qa` is a Arabic model originally trained by `salti`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_arabic_qa_ar_4.0.0_3.0_1655992517026.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_arabic_qa_ar_4.0.0_3.0_1655992517026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_large_arabic_qa","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_large_arabic_qa","ar") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.xlm_roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_large_arabic_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|ar| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/salti/xlm-roberta-large-arabic_qa --- layout: model title: Pipeline to Detect Adverse Drug Events (biobert) author: John Snow Labs name: ner_ade_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_ade_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_ade_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_ade_biobert_pipeline_en_4.3.0_3.2_1679311852288.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_ade_biobert_pipeline_en_4.3.0_3.2_1679311852288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_ade_biobert_pipeline", "en", "clinical/models") text = '''Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_ade_biobert_pipeline", "en", "clinical/models") val text = "Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps" val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biobert_ade.pipeline").predict("""Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:---------------|--------:|------:|:------------|-------------:| | 0 | Lipitor | 12 | 18 | DRUG | 0.9996 | | 1 | severe fatigue | 52 | 65 | ADE | 0.7588 | | 2 | voltaren | 97 | 104 | DRUG | 0.998 | | 3 | cramps | 152 | 157 | ADE | 0.9258 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ade_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering (from shahrukhx01) author: John Snow Labs name: roberta_qa_roberta_base_squad2_boolq_baseline date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-boolq-baseline` is a English model originally trained by `shahrukhx01`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_boolq_baseline_en_4.0.0_3.0_1655735101028.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_squad2_boolq_baseline_en_4.0.0_3.0_1655735101028.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_squad2_boolq_baseline","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_squad2_boolq_baseline","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_shahrukhx01").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_squad2_boolq_baseline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/shahrukhx01/roberta-base-squad2-boolq-baseline --- layout: model title: Part of Speech for German author: John Snow Labs name: pos_ud_hdt date: 2021-03-08 tags: [part_of_speech, open_source, german, pos_ud_hdt, de] task: Part of Speech Tagging language: de edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADP - DET - ADJ - NOUN - VERB - PRON - PROPN - X - PUNCT - CCONJ - NUM - ADV - AUX - SCONJ - PART - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/german/pretrained-german-models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_hdt_de_3.0.0_3.0_1615230160154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_hdt_de_3.0.0_3.0_1615230160154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols("sentence") \ .setOutputCol("token") posTagger = PerceptronModel.pretrained("pos_ud_hdt", "de") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) data = spark.createDataFrame([["Hallo aus John Snow Labs! "]], ["text"]) result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_hdt", "de") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hallo aus John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Hallo aus John Snow Labs!"] token_df = nlu.load('de.pos').predict(text) token_df ```

## Results ```bash token pos 0 Hallo NOUN 1 aus ADP 2 John PROPN 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_hdt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|de| --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465516 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465516` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465516_en_4.0.0_3.0_1655986218284.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465516_en_4.0.0_3.0_1655986218284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465516","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465516","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465516.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465516| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465516 --- layout: model title: Entity Recognizer LG author: John Snow Labs name: entity_recognizer_lg date: 2022-06-25 tags: [da, open_source] task: Named Entity Recognition language: da edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_da_4.0.0_3.0_1656129041439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_da_4.0.0_3.0_1656129041439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("entity_recognizer_lg", "da") result = pipeline.annotate("""I love johnsnowlabs! """) ``` {:.nlu-block} ```python import nlu nlu.load("da.ner.lg").predict("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|da| |Size:|2.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: English asr_wav2vec2_large_xls_r_300m_CN_colab TFWav2Vec2ForCTC from li666 author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_CN_colab date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_CN_colab` is a English model originally trained by li666. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_CN_colab_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_CN_colab_en_4.2.0_3.0_1664094455307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_CN_colab_en_4.2.0_3.0_1664094455307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_CN_colab", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_CN_colab", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_CN_colab| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Sentiment Analysis for Food Reviews author: John Snow Labs name: distilbert_base_sequence_classifier_food date: 2022-02-14 tags: [food, amazon, distilbert, sequence_classification, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: DistilBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` ([link](https://huggingface.co/Tejas3/distillbert_base_uncased_amazon_food_review_300)) and it's been trained on Amazon Food Review dataset, leveraging `Distil-BERT` embeddings and `DistilBertForSequenceClassification` for text classification purposes. The model classifies `Positive` or `Negative` sentiments of texts related to food reviews. ## Predicted Entities `Positive`, `Negative` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_food_en_3.4.0_3.0_1644846756022.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_sequence_classifier_food_en_3.4.0_3.0_1644846756022.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = DistilBertForSequenceClassification.pretrained("distilbert_base_sequence_classifier_food", "en")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = Pipeline(stages=[document_assembler, tokenizer, sequenceClassifier]) light_pipeline = LightPipeline(pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.annotate("The first time I ever used them was about 9 months ago. The food that came was just left at my front doorstep all over the place. Bread was smashed, bananas nearly rotten, and containers crushed. Given the weather, I decided to give them another try. This time my order was cancelled 6 hours before delivery. When cancelled, they didn't even give an explanation as to why it was cancelled. Amazon just needs to close up this portion of their shop.") ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = DistilBertForSequenceClassification.pretrained("distilbert_base_sequence_classifier_food", "en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val example = Seq.empty["The first time I ever used them was about 9 months ago. The food that came was just left at my front doorstep all over the place. Bread was smashed, bananas nearly rotten, and containers crushed. Given the weather, I decided to give them another try. This time my order was cancelled 6 hours before delivery. When cancelled, they didn't even give an explanation as to why it was cancelled. Amazon just needs to close up this portion of their shop."].toDS.toDF("text") val result = pipeline.fit(example1).transform(example) ```

## Results ```bash ['Negative'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_sequence_classifier_food| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[class]| |Language:|en| |Size:|249.8 MB| |Case sensitive:|true| |Max sentence length:|256| --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from efederici) author: John Snow Labs name: t5_it5_efficient_small_fanpage date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-fanpage` is a Italian model originally trained by `efederici`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_fanpage_it_4.3.0_3.0_1675103771407.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_fanpage_it_4.3.0_3.0_1675103771407.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_fanpage","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_fanpage","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_fanpage| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|593.8 MB| ## References - https://huggingface.co/efederici/it5-efficient-small-fanpage --- layout: model title: English XlmRoBertaForQuestionAnswering model (from deepset) author: John Snow Labs name: xlm_roberta_base_qa_squad2 date: 2022-06-15 tags: [open_source, xlmroberta, question_answering, en] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-squad2` is a English model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_qa_squad2_en_4.0.0_3.0_1655293726154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_qa_squad2_en_4.0.0_3.0_1655293726154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_base_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_base_qa_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_base_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|875.0 MB| |Case sensitive:|false| |Max sentence length:|512| ## References https://huggingface.co/deepset/xlm-roberta-base-squad2 --- layout: model title: English image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2 ViTForImageClassification from V0ltron author: John Snow Labs name: image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2` is a English model originally trained by V0ltron. ## Predicted Entities `FullMed`, `MiddleLetter`, `BoundaryLetter`, `BoundaryMed`, `MiddleMed`, `FullLetter` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2_en_4.1.0_3.0_1660165490741.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2_en_4.1.0_3.0_1660165490741.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_finetuned_largerDataSet_docSeperator_more_labels_all_apache2| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Publicity Clause Binary Classifier author: John Snow Labs name: legclf_publicity_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `publicity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `publicity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_publicity_clause_en_1.0.0_3.2_1660122871432.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_publicity_clause_en_1.0.0_3.2_1660122871432.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_publicity_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[publicity]| |[other]| |[other]| |[publicity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_publicity_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.98 0.96 96 publicity 0.94 0.84 0.89 37 accuracy - - 0.94 133 macro-avg 0.94 0.91 0.92 133 weighted-avg 0.94 0.94 0.94 133 ``` --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from hackathon-pln-es) author: John Snow Labs name: roberta_qa_base_bne_squad2 date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-squad2-es` is a Spanish model originally trained by `hackathon-pln-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_bne_squad2_es_4.2.4_3.0_1669985982782.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_bne_squad2_es_4.2.4_3.0_1669985982782.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_squad2","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_squad2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_bne_squad2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|456.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/hackathon-pln-es/roberta-base-bne-squad2-es --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Qiliang) author: John Snow Labs name: distilbert_qa_qiliang_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Qiliang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_qiliang_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768989901.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_qiliang_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768989901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_qiliang_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_qiliang_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_qiliang_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Qiliang/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-256-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2_en_4.0.0_3.0_1657184713981.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2_en_4.0.0_3.0_1657184713981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_256_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-256-finetuned-squad-seed-2 --- layout: model title: Medical Question Answering (biogpt_pubmed_qa) author: John Snow Labs name: biogpt_pubmed_qa date: 2023-03-09 tags: [licensed, clinical, en, gpt, biogpt, pubmed, question_answering, tensorflow] task: Question Answering language: en edition: Healthcare NLP 4.3.1 spark_version: 3.0 published: false engine: tensorflow annotator: MedicalQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model has been trained with medical documents and can generate two types of answers, short and long. Types of questions are supported: `"short"` (producing yes/no/maybe) answers and `"full"` (long answers). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/biogpt_pubmed_qa_en_4.3.1_3.0_1678372297946.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/biogpt_pubmed_qa_en_4.3.1_3.0_1678372297946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_indebtedness_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[indebtedness]| |[other]| |[other]| |[indebtedness]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indebtedness_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support indebtedness 0.95 0.88 0.91 24 other 0.96 0.98 0.97 66 accuracy - - 0.96 90 macro-avg 0.96 0.93 0.94 90 weighted-avg 0.96 0.96 0.95 90 ``` --- layout: model title: English asr_wav2vec2_25_1Aug_2022 TFWav2Vec2ForCTC from Roshana author: John Snow Labs name: asr_wav2vec2_25_1Aug_2022 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_25_1Aug_2022` is a English model originally trained by Roshana. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_25_1Aug_2022_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_25_1Aug_2022_en_4.2.0_3.0_1664115989562.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_25_1Aug_2022_en_4.2.0_3.0_1664115989562.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_25_1Aug_2022", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_25_1Aug_2022", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_25_1Aug_2022| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-2` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2_en_4.0.0_3.0_1654191719402.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2_en_4.0.0_3.0_1654191719402.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_64d_seed_2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|378.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-2 --- layout: model title: Stopwords Remover for English language (306 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, en, open_source] task: Stop Words Removal language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_en_3.4.1_3.0_1646672348840.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_en_3.4.1_3.0_1646672348840.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------+ |result | +--------+ |[better]| +--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|en| |Size:|2.1 KB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from cgou) author: John Snow Labs name: roberta_qa_fin_v1_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fin_RoBERTa-v1-finetuned-squad` is a English model originally trained by `cgou`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_fin_v1_finetuned_squad_en_4.3.0_3.0_1674210819999.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_fin_v1_finetuned_squad_en_4.3.0_3.0_1674210819999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fin_v1_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_fin_v1_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fin_v1_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|248.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/cgou/fin_RoBERTa-v1-finetuned-squad --- layout: model title: Translate Uralic languages to English Pipeline author: John Snow Labs name: translate_urj_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, urj, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `urj` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_urj_en_xx_2.7.0_2.4_1609699134258.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_urj_en_xx_2.7.0_2.4_1609699134258.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_urj_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_urj_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.urj.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_urj_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_lotr ViTForImageClassification from jjhoffstein author: John Snow Labs name: image_classifier_vit_lotr date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_lotr` is a English model originally trained by jjhoffstein. ## Predicted Entities `gandalf`, `gollum`, `aragorn`, `frodo`, `legolas` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lotr_en_4.1.0_3.0_1660168933925.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lotr_en_4.1.0_3.0_1660168933925.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_lotr", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_lotr", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_lotr| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Tagalog RoBERTa Embeddings (Large) author: John Snow Labs name: roberta_embeddings_roberta_tagalog_large date: 2022-04-14 tags: [roberta, embeddings, tl, open_source] task: Embeddings language: tl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-tagalog-large` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_tagalog_large_tl_3.4.2_3.0_1649948937589.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_tagalog_large_tl_3.4.2_3.0_1649948937589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_tagalog_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|tl| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/jcblaise/roberta-tagalog-large - https://blaisecruz.com --- layout: model title: Named Entity Recognition (NER) Model in Finnish (GloVe 840B 300) author: John Snow Labs name: finnish_ner_840B_300 date: 2020-09-01 task: Named Entity Recognition language: fi edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [ner, fi, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Finnish NER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. The model is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Product-`PRO`, Date-`DATE`, Event-`EVENT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/PP_EXPLAIN_DOCUMENT_FI/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/PP_EXPLAIN_DOCUMENT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/finnish_ner_840B_300_fi_2.6.0_2.4_1598965807720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/finnish_ner_840B_300_fi_2.6.0_2.4_1598965807720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PER | |lokakuuta 1955 |DATE | |ulkoministeriöitä |ORG | |Microsoft |ORG | |Microsoft Corporationin|ORG | |Microsoft |ORG | |Gates |PER | |maj 2014 |DATE | |1970'erne 1980 1980erne|DATE | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |1975 |DATE | |Albuquerque |LOC | |New Mexico |LOC | |Det |LOC | |Gates |PER | |tammikuu 2000 |DATE | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finnish_ner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.6.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|fi| |Case sensitive:|false| {:.h2_title} ## Data Source The detailed information can be found from [https://www.aclweb.org/anthology/2020.lrec-1.567.pdf](https://www.aclweb.org/anthology/2020.lrec-1.567.pdf) --- layout: model title: Japanese Electra Embeddings (from izumi-lab) author: John Snow Labs name: electra_embeddings_electra_small_paper_japanese_generator date: 2022-05-17 tags: [ja, open_source, electra, embeddings] task: Embeddings language: ja edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-small-paper-japanese-generator` is a Japanese model orginally trained by `izumi-lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_small_paper_japanese_generator_ja_3.4.4_3.0_1652786712318.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_small_paper_japanese_generator_ja_3.4.4_3.0_1652786712318.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_small_paper_japanese_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ja| |Size:|19.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/izumi-lab/electra-small-paper-japanese-generator - https://github.com/google-research/electra - https://github.com/retarfi/language-pretraining/tree/v1.0 - https://arxiv.org/abs/2003.10555 - https://arxiv.org/abs/2003.10555 - https://creativecommons.org/licenses/by-sa/4.0/ --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from tabo) author: John Snow Labs name: distilbert_qa_tabo_base_uncased_finetuned_squad2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad2` is a English model originally trained by `tabo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_tabo_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773599663.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_tabo_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773599663.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tabo_base_uncased_finetuned_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tabo_base_uncased_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_tabo_base_uncased_finetuned_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tabo/distilbert-base-uncased-finetuned-squad2 --- layout: model title: Legal Demand Registration Clause Binary Classifier author: John Snow Labs name: legclf_demand_registration_clause date: 2022-12-07 tags: [en, legal, classification, clauses, demand_registration, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `demand-registration` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `demand-registration`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_demand_registration_clause_en_1.0.0_3.0_1670444564459.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_demand_registration_clause_en_1.0.0_3.0_1670444564459.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_demand_registration_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[demand-registration]| |[other]| |[other]| |[demand-registration]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_demand_registration_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support demand-registration 1.00 0.96 0.98 28 other 0.99 1.00 0.99 73 accuracy - - 0.99 101 macro-avg 0.99 0.98 0.99 101 weighted-avg 0.99 0.99 0.99 101 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from google) author: John Snow Labs name: t5_flan_base date: 2023-03-01 tags: [open_source, t5, flan, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. flan-t5-base is a English model originally trained by google. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_flan_base_xx_4.3.0_3.0_1677702524850.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_flan_base_xx_4.3.0_3.0_1677702524850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_flan_base","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_flan_base","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_flan_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|1.0 GB| ## References https://huggingface.co/google/flan-t5-base --- layout: model title: Legal Change In Control Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_change_in_control_bert date: 2023-03-05 tags: [en, legal, classification, clauses, change_in_control, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Change_In_Control` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Change_In_Control`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_change_in_control_bert_en_1.0.0_3.0_1678050687048.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_change_in_control_bert_en_1.0.0_3.0_1678050687048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_change_in_control_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Change_In_Control]| |[Other]| |[Other]| |[Change_In_Control]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_change_in_control_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Change_In_Control 0.95 0.97 0.96 38 Other 0.98 0.97 0.98 61 accuracy - - 0.97 99 macro-avg 0.97 0.97 0.97 99 weighted-avg 0.97 0.97 0.97 99 ``` --- layout: model title: Fast Neural Machine Translation Model from Tiv to English author: John Snow Labs name: opus_mt_tiv_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tiv, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tiv` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tiv_en_xx_2.7.0_2.4_1609167406296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tiv_en_xx_2.7.0_2.4_1609167406296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tiv_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Financial 10K Filings NER (Summary page, SEC embeddings) author: John Snow Labs name: finner_sec_10k_summary date: 2022-10-25 tags: [sec, 10k, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true recommended: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole financial report. Instead: - Split by paragraphs; - Use the `finclf_form_10k_summary_item` Text Classifier to select only these paragraps; This Financial NER Model is aimed to process the first summary page of 10K filings and extract the information about the Company submitting the filing, trading data, address / phones, CFN, IRS, etc. This is another version of `finner_10k_summary` trained with better financial embeddings (`bert_embeddings_sec_bert_base`) ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_sec_10k_summary_en_1.0.0_3.0_1666711517681.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_sec_10k_summary_en_1.0.0_3.0_1666711517681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") \ .setCustomBounds(["\n\n"]) tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_sec_10k_summary","en","finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES AND EXCHANGE ACT OF 1934 For the annual period ended January 31, 2021 or TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 For the transition period from________to_______ Commission File Number: 001-38856 PAGERDUTY, INC. (Exact name of registrant as specified in its charter) Delaware 27-2793871 (State or other jurisdiction of incorporation or organization) (I.R.S. Employer Identification Number) 600 Townsend St., Suite 200, San Francisco, CA 94103 (844) 800-3889 (Address, including zip code, and telephone number, including area code, of registrant’s principal executive offices) Securities registered pursuant to Section 12(b) of the Act: Title of each class Trading symbol(s) Name of each exchange on which registered Common Stock, $0.000005 par value, PD New York Stock Exchange"""]]).toDF("text") result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("ticker"), F.expr("cols['1']['entity']").alias("label")).show(50, truncate = False) ```

## Results ```bash +----------------------------------------------+-----------------+ |ticker |label | +----------------------------------------------+-----------------+ |January 31, 2021 |FISCAL_YEAR | |001-38856 |CFN | |PAGERDUTY, INC |ORG | |Delaware |STATE | |27-2793871 |IRS | |600 Townsend St., Suite 200, San Francisco, CA|ADDRESS | |(844) 800-3889 |PHONE | |Common Stock |TITLE_CLASS | |$0.000005 |TITLE_CLASS_VALUE| |PD |TICKER | |New York Stock Exchange |STOCK_EXCHANGE | +----------------------------------------------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_sec_10k_summary| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.5 MB| ## References Manual annotations on 10-K Filings ## Benchmarking ```bash label precision recall f1-score support B-TITLE_CLASS 1.00 1.00 1.00 15 I-TITLE_CLASS 1.00 1.00 1.00 21 B-ORG 0.84 0.66 0.74 62 I-ORG 0.88 0.76 0.82 93 B-STOCK_EXCHANGE 0.86 0.86 0.86 14 I-STOCK_EXCHANGE 0.98 0.98 0.98 50 B-PHONE 0.95 0.87 0.91 23 I-PHONE 0.95 1.00 0.98 60 B-STATE 0.89 0.85 0.87 20 B-IRS 1.00 0.88 0.93 16 B-ADDRESS 0.94 0.83 0.88 18 I-ADDRESS 0.92 0.97 0.94 144 B-TICKER 0.86 0.92 0.89 13 B-FISCAL_YEAR 0.96 0.88 0.92 50 I-FISCAL_YEAR 0.93 0.92 0.92 125 B-TITLE_CLASS_VALUE 1.00 0.93 0.97 15 B-CFN 0.92 1.00 0.96 12 micro-avg 0.93 0.89 0.91 751 macro-avg 0.84 0.81 0.82 751 weighted-avg 0.92 0.89 0.91 751 ``` --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_company_all_903429540 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-company_all-903429540` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Company`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_all_903429540_en_4.3.1_3.0_1678134087932.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_all_903429540_en_4.3.1_3.0_1678134087932.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_all_903429540","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_all_903429540","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_company_all_903429540| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-company_all-903429540 --- layout: model title: Mapping SNOMED Codes with Their Corresponding UMLS Codes author: John Snow Labs name: snomed_umls_mapper date: 2022-06-27 tags: [snomed, umls, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps SNOMED codes to corresponding UMLS codes under the Unified Medical Language System (UMLS). ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapper_en_3.5.3_3.0_1656313105018.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapper_en_3.5.3_3.0_1656313105018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbert_jsl_medium_uncased", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") snomed_resolver = SentenceEntityResolverModel\ .pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("snomed_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel\ .pretrained("snomed_umls_mapper", "en", "clinical/models") \ .setInputCols(["snomed_code"])\ .setOutputCol("umls_mappings")\ .setRels(["umls_code"]) pipeline = Pipeline(stages = [ documentAssembler, sbert_embedder, snomed_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline= LightPipeline(model) result = light_pipeline.fullAnnotate("Radiating chest pain") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbert_jsl_medium_uncased", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val snomed_resolver = SentenceEntityResolverModel .pretrained("sbertresolve_snomed_conditions", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("snomed_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel .pretrained("snomed_umls_mapper", "en", "clinical/models") .setInputCols("snomed_code") .setOutputCol("umls_mappings") .setRels(Array("umls_code")) val pipeline = new Pipeline(stages = Array( documentAssembler, sbert_embedder, snomed_resolver, chunkerMapper )) val data = Seq("Radiating chest pain").toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.snomed_to_umls").predict("""Radiating chest pain""") ```

## Results ```bash | | ner_chunk | snomed_code | umls_mappings | |---:|:---------------------|--------------:|:----------------| | 0 | Radiating chest pain | 10000006 | C0232289 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_umls_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[snomed_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|5.1 MB| --- layout: model title: Part of Speech for Russian author: John Snow Labs name: pos_ud_gsd date: 2020-03-12 13:48:00 +0800 task: Part of Speech Tagging language: ru edition: Spark NLP 2.4.4 spark_version: 2.4 tags: [pos, ru] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ru_2.4.4_2.4_1584013495069.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ru_2.4.4_2.4_1584013495069.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_gsd", "ru") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_gsd", "ru") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Помимо того, что он король севера, Джон Сноу - английский врач и лидер в разработке анестезии и медицинской гигиены."""] pos_df = nlu.load('ru.pos.ud_gsd').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=5, result='ADP', metadata={'word': 'Помимо'}), Row(annotatorType='pos', begin=7, end=10, result='PRON', metadata={'word': 'того'}), Row(annotatorType='pos', begin=11, end=11, result='PUNCT', metadata={'word': ','}), Row(annotatorType='pos', begin=13, end=15, result='SCONJ', metadata={'word': 'что'}), Row(annotatorType='pos', begin=17, end=18, result='PRON', metadata={'word': 'он'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Type:|pos| |Compatibility:|Spark NLP 2.4.4| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|ru| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Sentence Entity Resolver for Snomed Aux Concepts, CT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_auxConcepts language: en nav_key: models repository: clinical/models date: 2020-11-27 task: Entity Resolution edition: Healthcare NLP 2.6.4 spark_version: 2.4 tags: [clinical,entity_resolution,en] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model maps extracted medical entities to Snomed codes (with Morph Abnormality, Procedure, Substance, Physical Object, Body Structure concepts from CT version) using chunk embeddings. {:.h2_title} ## Predicted Entities Snomed Codes and their normalized definition with ``sbiobert_base_cased_mli`` embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_en_2.6.4_2.4_1606235765319.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_en_2.6.4_2.4_1606235765319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") snomed_aux_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_snomed_auxConcepts","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = model.fit(data).transform(data) ``` ```scala ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val snomed_aux_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_auxConcepts","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_resolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 73578008| 0.5321|hyperdistension::...|73578008:::147849...| |chronic renal ins...| 83|109| PROBLEM|151137006| 0.3356|renal function ob...|151137006:::26681...| | COPD| 113|116| PROBLEM|395159008| 0.1374|chronic obstructi...|395159008:::39059...| | gastritis| 120|128| PROBLEM| 57632009| 0.1274|gastroplication::...|57632009:::216090...| | TIA| 136|138| PROBLEM|449758002| 0.1984|traumatic infarct...|449758002:::85844...| |a non-ST elevatio...| 182|202| PROBLEM|713264002| 0.0941|nontraumatic rupt...|713264002:::31036...| |Guaiac positive s...| 208|229| PROBLEM| 25580003| 0.0906|faecaloma:::faeca...|25580003:::891580...| |cardiac catheteri...| 295|317| TEST| 41976001| 0.4957|cardiac catheteri...|41976001:::141945...| | PTCA| 324|327|TREATMENT|309817004| 0.0660|pulmonary angiogr...|309817004:::31264...| | mid LAD lesion| 332|345| PROBLEM|193467007| 0.1213|mid portion of an...|193467007:::91748...| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_snomed_auxConcepts | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.4 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on SNOMED (CT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: Hindi Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:34:00 +0800 task: Lemmatization language: hi edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, hi] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_hi_2.5.5_2.4_1596054396201.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_hi_2.5.5_2.4_1596054396201.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "hi") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "hi") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।"""] lemma_df = nlu.load('hi.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='उत्तर', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=6, end=7, result='विपिन', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=9, end=12, result='राजा', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=14, end=17, result='हो', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=19, end=20, result='विपिन', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|hi| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from dspg) author: John Snow Labs name: distilbert_qa_dspg_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `dspg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_dspg_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770583896.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_dspg_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770583896.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_dspg_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_dspg_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_dspg_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/dspg/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForTokenClassification Cased model (from Shenzy2) author: John Snow Labs name: bert_token_classifier_autotrain_tk_1181244086 date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-tk-1181244086` is a English model originally trained by `Shenzy2`. ## Predicted Entities `design_concept`, `ui_element` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_tk_1181244086_en_4.2.4_3.0_1669814679080.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_tk_1181244086_en_4.2.4_3.0_1669814679080.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_tk_1181244086","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_tk_1181244086","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autotrain_tk_1181244086| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Shenzy2/autotrain-tk-1181244086 --- layout: model title: Part of Speech for Indonesian author: John Snow Labs name: pos_ud_gsd date: 2020-07-29 23:34:00 +0800 task: Part of Speech Tagging language: id edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [pos, id] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_id_2.5.5_2.4_1596054136894.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_id_2.5.5_2.4_1596054136894.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_gsd", "id") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_gsd", "id") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Selain menjadi raja utara, John Snow adalah seorang dokter Inggris dan pemimpin dalam pengembangan anestesi dan kebersihan medis."""] pos_df = nlu.load('id.pos').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=5, result='ADP', metadata={'word': 'Selain'}), Row(annotatorType='pos', begin=7, end=13, result='VERB', metadata={'word': 'menjadi'}), Row(annotatorType='pos', begin=15, end=18, result='NOUN', metadata={'word': 'raja'}), Row(annotatorType='pos', begin=20, end=24, result='NOUN', metadata={'word': 'utara'}), Row(annotatorType='pos', begin=25, end=25, result='PUNCT', metadata={'word': ','}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Type:|pos| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|id| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_el4 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-el4` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el4_en_4.3.0_3.0_1675111271789.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el4_en_4.3.0_3.0_1675111271789.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_el4","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_el4","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_el4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|366.4 MB| ## References - https://huggingface.co/google/t5-efficient-base-el4 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: German NER for Laws author: John Snow Labs name: legner_courts date: 2022-10-02 tags: [de, legal, ner, laws, court, licensed] task: Named Entity Recognition language: de edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect legal entities in German text, predicting up to 19 different labels: ``` | tag | meaning ----------------- | AN | Anwalt | EUN | Europäische Norm | GS | Gesetz | GRT | Gericht | INN | Institution | LD | Land | LDS | Landschaft | LIT | Literatur | MRK | Marke | ORG | Organisation | PER | Person | RR | Richter | RS | Rechtssprechung | ST | Stadt | STR | Straße | UN | Unternehmen | VO | Verordnung | VS | Vorschrift | VT | Vertrag ``` German Named Entity Recognition model, trained using a Deep Learning architecture (CharCNN + LSTM) with a Court Decisions (2017-2018) dataset (check `Data Source` section). You can also find transformer-based versions of this model in our Models Hub (`legner_bert_base_courts` and `legner_bert_large_courts`) ## Predicted Entities `STR`, `LIT`, `PER`, `EUN`, `VT`, `MRK`, `INN`, `UN`, `RS`, `ORG`, `GS`, `VS`, `LDS`, `GRT`, `VO`, `RR`, `LD`, `AN`, `ST` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_LEGAL_DE/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_courts_de_1.0.0_3.0_1664706079878.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_courts_de_1.0.0_3.0_1664706079878.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... word_embeddings = nlp.WordEmbeddingsModel.pretrained("w2v_cc_300d",'de','clinical/models')\ .setInputCols(["sentence", 'token'])\ .setOutputCol("embeddings")\ .setCaseSensitive(False) legal_ner = legal.NerModel.pretrained("legner_courts",'de','legal/models') \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... legal_pred_pipeline = nlp.Pipeline(stages = [document_assembler, sentence_detector, tokenizer, word_embeddings, legal_ner, ner_converter]) legal_light_model = LightPipeline(legal_pred_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = legal_light_model.fullAnnotate('''Jedoch wird der Verkehr darin naheliegend den Namen eines der bekanntesten Flüsse Deutschlands erkennen, welcher als Seitenfluss des Rheins durch Oberfranken, Unterfranken und Südhessen fließt und bei Mainz in den Rhein mündet. Klein , in : Maunz / Schmidt-Bleibtreu / Klein / Bethge , BVerfGG , § 19 Rn. 9 Richtlinien zur Bewertung des Grundvermögens – BewRGr – vom19. I September 1966 (BStBl I, S.890) ''') ```

## Results ```bash +---+---------------------------------------------------+----------+ | # | Chunks | Entities | +---+---------------------------------------------------+----------+ | 0 | Deutschlands | LD | +---+---------------------------------------------------+----------+ | 1 | Rheins | LDS | +---+---------------------------------------------------+----------+ | 2 | Oberfranken | LDS | +---+---------------------------------------------------+----------+ | 3 | Unterfranken | LDS | +---+---------------------------------------------------+----------+ | 4 | Südhessen | LDS | +---+---------------------------------------------------+----------+ | 5 | Mainz | ST | +---+---------------------------------------------------+----------+ | 6 | Rhein | LDS | +---+---------------------------------------------------+----------+ | 7 | Klein , in : Maunz / Schmidt-Bleibtreu / Klein... | LIT | +---+---------------------------------------------------+----------+ | 8 | Richtlinien zur Bewertung des Grundvermögens –... | VS | +---+---------------------------------------------------+----------+ | 9 | I September 1966 (BStBl I, S.890) | VS | +---+---------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_courts| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| |Size:|15.0 MB| ## References The dataset used to train this model is taken from Leitner, et.al (2019) Leitner, E., Rehm, G., and Moreno-Schneider, J. (2019). Fine-grained Named Entity Recognition in Legal Documents. In Maribel Acosta, et al., editors, Semantic Systems. The Power of AI and Knowledge Graphs. Proceedings of the 15th International Conference (SEMANTiCS2019), number 11702 in Lecture Notes in Computer Science, pages 272–287, Karlsruhe, Germany, 9. Springer. 10/11 September 2019. Source of the annotated text: Court decisions from 2017 and 2018 were selected for the dataset, published online by the Federal Ministry of Justice and Consumer Protection. The documents originate from seven federal courts: Federal Labour Court (BAG), Federal Fiscal Court (BFH), Federal Court of Justice (BGH), Federal Patent Court (BPatG), Federal Social Court (BSG), Federal Constitutional Court (BVerfG) and Federal Administrative Court (BVerwG). ## Benchmarking ```bash label prec rec f1 Macro-average 0.9210195 0.9186192 0.9198177 Micro-average 0.9833763 0.9837547 0.9835655 ``` --- layout: model title: Legal Agricultural Activity Document Classifier (EURLEX) author: John Snow Labs name: legclf_agricultural_activity_bert date: 2023-03-06 tags: [en, legal, classification, clauses, agricultural_activity, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_agricultural_activity_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Agricultural_Activity or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Agricultural_Activity`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agricultural_activity_bert_en_1.0.0_3.0_1678111704283.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agricultural_activity_bert_en_1.0.0_3.0_1678111704283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_agricultural_activity_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Agricultural_Activity]| |[Other]| |[Other]| |[Agricultural_Activity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agricultural_activity_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.0 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Agricultural_Activity 0.83 0.88 0.86 804 Other 0.86 0.79 0.82 694 accuracy - - 0.84 1498 macro-avg 0.84 0.84 0.84 1498 weighted-avg 0.84 0.84 0.84 1498 ``` --- layout: model title: Translate Xhosa to English Pipeline author: John Snow Labs name: translate_xh_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, xh, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `xh` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_xh_en_xx_2.7.0_2.4_1609690334845.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_xh_en_xx_2.7.0_2.4_1609690334845.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_xh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_xh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.xh.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_xh_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-4_H-256_A-4_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna_en_4.0.0_3.0_1654185276053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna_en_4.0.0_3.0_1654185276053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.uncased_4l_256d_a4a_256d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_256_A_4_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|42.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-256_A-4_squad2_covid-qna --- layout: model title: Pipeline to Detect PHI in Text author: John Snow Labs name: ner_deid_sd_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_sd](https://nlp.johnsnowlabs.com/2021/04/01/ner_deid_sd_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_pipeline_en_3.4.1_3.0_1647869878449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_pipeline_en_3.4.1_3.0_1647869878449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_sd_pipeline", "en", "clinical/models") pipeline.annotate("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_sd_pipeline", "en", "clinical/models") pipeline.annotate("A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.sd.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash +-----------------------------+--------+ |chunks |entities| +-----------------------------+--------+ |2093-01-13 |DATE | |David Hale |NAME | |Hendrickson, Ora |NAME | |7194334 |ID | |01/13/93 |DATE | |Oliveira |NAME | |1-11-2000 |DATE | |Cocke County Baptist Hospital|LOCATION| |0295 Keats Street |LOCATION| |(302) 786-5227 |CONTACT | |Brothers Coal-Mine |LOCATION| +-----------------------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_sd_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English BertForQuestionAnswering model (from MichelBartels) author: John Snow Labs name: bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tinybert-6l-768d-squad2-large-teacher-finetuned-step1` is a English model orginally trained by `MichelBartels`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1_en_4.0.0_3.0_1654192496308.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1_en_4.0.0_3.0_1654192496308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_tiny_768d_v2.by_MichelBartels").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_step1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|249.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MichelBartels/tinybert-6l-768d-squad2-large-teacher-finetuned-step1 --- layout: model title: Modern Greek (1453-) asr_xlsr_53_wav2vec_greek TFWav2Vec2ForCTC from harshit345 author: John Snow Labs name: pipeline_asr_xlsr_53_wav2vec_greek date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_53_wav2vec_greek` is a Modern Greek (1453-) model originally trained by harshit345. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xlsr_53_wav2vec_greek_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_53_wav2vec_greek_el_4.2.0_3.0_1664109717228.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_53_wav2vec_greek_el_4.2.0_3.0_1664109717228.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xlsr_53_wav2vec_greek', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xlsr_53_wav2vec_greek", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xlsr_53_wav2vec_greek| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate English to Hungarian Pipeline author: John Snow Labs name: translate_en_hu date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, hu, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `hu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_hu_xx_2.7.0_2.4_1609700744893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_hu_xx_2.7.0_2.4_1609700744893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_hu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_hu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.hu').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_hu| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: French CamemBert Embeddings (from jcai1) author: John Snow Labs name: camembert_embeddings_jcai1_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `jcai1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_jcai1_generic_model_fr_3.4.4_3.0_1653988792605.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_jcai1_generic_model_fr_3.4.4_3.0_1653988792605.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_jcai1_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/jcai1/dummy-model --- layout: model title: Japanese Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_large_japanese_unidic_luw_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, ja, open_source] task: Part of Speech Tagging language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-japanese-unidic-luw-upos` is a Japanese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_japanese_unidic_luw_upos_ja_3.4.2_3.0_1652092222361.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_japanese_unidic_luw_upos_ja_3.4.2_3.0_1652092222361.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_large_japanese_unidic_luw_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ja| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-large-japanese-unidic-luw-upos - https://universaldependencies.org/u/pos/ - https://pypi.org/project/fugashi - https://pypi.org/project/unidic-lite - https://pypi.org/project/pytokenizations - http://id.nii.ac.jp/1001/00216223/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Legal Loan agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_loan_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_loan_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class loan-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `loan-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_loan_agreement_en_1.0.0_3.0_1666620991453.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_loan_agreement_en_1.0.0_3.0_1666620991453.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_loan_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[loan-agreement]| |[other]| |[other]| |[loan-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_loan_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support loan-agreement 0.92 0.92 0.92 39 other 0.95 0.95 0.95 62 accuracy - - 0.94 101 macro-avg 0.94 0.94 0.94 101 weighted-avg 0.94 0.94 0.94 101 ``` --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6_en_4.0.0_3.0_1655730927446.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6_en_4.0.0_3.0_1655730927446.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_1024d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|439.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-6 --- layout: model title: French CamemBert Embeddings (from ankitkupadhyay) author: John Snow Labs name: camembert_embeddings_ankitkupadhyay_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `ankitkupadhyay`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ankitkupadhyay_generic_model_fr_3.4.4_3.0_1653987503170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ankitkupadhyay_generic_model_fr_3.4.4_3.0_1653987503170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_ankitkupadhyay_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/ankitkupadhyay/dummy-model --- layout: model title: Pipeline to find clinical events and find temporal relations (ERA) author: John Snow Labs name: explain_clinical_doc_era date: 2021-04-01 tags: [pipeline, en, licensed, clinical] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline with `ner_clinical_events`, `assertion_dl` and `re_temporal_events_clinical`. It will extract clinical entities, assign assertion status and find temporal relationships between clinical entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_3.0.0_3.0_1617297404938.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_3.0.0_3.0_1617297404938.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python era_pipeline = PretrainedPipeline('explain_clinical_doc_era', 'en', 'clinical/models') annotations = era_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)[0] ``` ```scala val era_pipeline = new PretrainedPipeline("explain_clinical_doc_era", "en", "clinical/models") val result = era_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.era").predict("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """) ```

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|:-----------|:--------------|----------------:|--------------:|:--------------------------|:--------------|----------------:|--------------:|:------------------------------|-------------:| | 0 | AFTER | OCCURRENCE | 7 | 14 | admitted | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | 0.963836 | | 1 | BEFORE | OCCURRENCE | 7 | 14 | admitted | DATE | 45 | 54 | 2 days ago | 0.587098 | | 2 | BEFORE | OCCURRENCE | 7 | 14 | admitted | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.999991 | | 3 | OVERLAP | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | DATE | 45 | 54 | 2 days ago | 0.996056 | | 4 | BEFORE | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.995216 | | 5 | OVERLAP | DATE | 45 | 54 | 2 days ago | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.996954 | | 6 | BEFORE | EVIDENTIAL | 119 | 124 | denied | PROBLEM | 126 | 129 | pain | 1 | | 7 | BEFORE | EVIDENTIAL | 119 | 124 | denied | PROBLEM | 135 | 146 | any headache | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_era| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Typed Dependency Parsing pipeline for English author: John Snow Labs name: dependency_parse date: 2022-06-29 tags: [pipeline, dependency_parsing, untyped_dependency_parsing, typed_dependency_parsing, laballed_depdency_parsing, unlaballed_depdency_parsing, en, open_source] task: Dependency Parser language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: Pipeline article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Typed Dependency parser, trained on the on the CONLL dataset. Dependency parsing is the task of extracting a dependency parse of a sentence that represents its grammatical structure and defines the relationships between “head” words and words, which modify those heads. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dependency_parse_en_4.0.0_3.0_1656456276940.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dependency_parse_en_4.0.0_3.0_1656456276940.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------------------------------------+--------------------------------------------------------+ |result |result | +---------------------------------------------------------------------------------+--------------------------------------------------------+ |[ROOT, Dependencies, represents, words, relationships, Sentence, Sentence, words]|[root, parataxis, nsubj, amod, nsubj, case, nsubj, flat]| +---------------------------------------------------------------------------------+--------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dependency_parse| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|24.1 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - DependencyParserModel - TypedDependencyParserModel --- layout: model title: Legal Agreement Of Purchase And Sale Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_agreement_of_purchase_and_sale_bert date: 2023-01-26 tags: [en, legal, classification, agreement, purchase, sale, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_agreement_of_purchase_and_sale_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `agreement-of-purchase-and-sale` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `agreement-of-purchase-and-sale`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agreement_of_purchase_and_sale_bert_en_1.0.0_3.0_1674731506904.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agreement_of_purchase_and_sale_bert_en_1.0.0_3.0_1674731506904.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_agreement_of_purchase_and_sale_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[agreement-of-purchase-and-sale]| |[other]| |[other]| |[agreement-of-purchase-and-sale]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agreement_of_purchase_and_sale_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support agreement-of-purchase-and-sale 0.96 0.98 0.97 51 other 0.99 0.98 0.99 116 accuracy - - 0.98 167 macro-avg 0.98 0.98 0.98 167 weighted-avg 0.98 0.98 0.98 167 ``` --- layout: model title: English asr_wav2vec2_base_100h_test TFWav2Vec2ForCTC from saahith author: John Snow Labs name: asr_wav2vec2_base_100h_test date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_test` is a English model originally trained by saahith. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_100h_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_test_en_4.2.0_3.0_1664094915243.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_test_en_4.2.0_3.0_1664094915243.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_100h_test", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_100h_test", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_100h_test| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_spanbert_finetuned_squadv1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-finetuned-squadv1` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_finetuned_squadv1_en_4.0.0_3.0_1654191785442.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_finetuned_squadv1_en_4.0.0_3.0_1654191785442.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_finetuned_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|402.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/spanbert-finetuned-squadv1 - https://arxiv.org/abs/1907.10529 - https://twitter.com/mrm8488 - https://github.com/facebookresearch - https://github.com/facebookresearch/SpanBERT - https://github.com/facebookresearch/SpanBERT#pre-trained-models - https://rajpurkar.github.io/SQuAD-explorer/ - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: Fast Neural Machine Translation Model from English to Afrikaans author: John Snow Labs name: opus_mt_en_af date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, af, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `af` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_af_xx_2.7.0_2.4_1609164103105.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_af_xx_2.7.0_2.4_1609164103105.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_af| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Sourabh714) author: John Snow Labs name: distilbert_qa_sourabh714_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Sourabh714`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sourabh714_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769323017.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sourabh714_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769323017.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sourabh714_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sourabh714_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sourabh714_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Sourabh714/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from atoivat) author: John Snow Labs name: distilbert_qa_atoivat_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `atoivat`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_atoivat_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770083162.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_atoivat_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770083162.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_atoivat_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_atoivat_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_atoivat_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/atoivat/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from msms) author: John Snow Labs name: distilbert_qa_msms_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `msms`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_msms_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772240858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_msms_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772240858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_msms_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_msms_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_msms_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/msms/distilbert-base-uncased-finetuned-squad --- layout: model title: Abkhazian asr_xls_r_ab_test_by_baaastien TFWav2Vec2ForCTC from baaastien author: John Snow Labs name: pipeline_asr_xls_r_ab_test_by_baaastien date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_baaastien` is a Abkhazian model originally trained by baaastien. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xls_r_ab_test_by_baaastien_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_ab_test_by_baaastien_ab_4.2.0_3.0_1664020833113.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_ab_test_by_baaastien_ab_4.2.0_3.0_1664020833113.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_r_ab_test_by_baaastien', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_r_ab_test_by_baaastien", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_r_ab_test_by_baaastien| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|448.1 KB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Recognize Entities DL Pipeline for Dutch - Small author: John Snow Labs name: entity_recognizer_sm date: 2021-03-22 tags: [open_source, dutch, entity_recognizer_sm, pipeline, nl] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: nl edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_nl_3.0.0_3.0_1616442569062.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_nl_3.0.0_3.0_1616442569062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_sm', lang = 'nl') annotations = pipeline.fullAnnotate(""Hallo van John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_sm", lang = "nl") val result = pipeline.fullAnnotate("Hallo van John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hallo van John Snow Labs! ""] result_df = nlu.load('nl.ner').predict(text) result_df ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_rnd_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_rnd_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.rnd.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_rnd_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_dm256 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-dm256` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dm256_en_4.3.0_3.0_1675119242272.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dm256_en_4.3.0_3.0_1675119242272.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_dm256","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_dm256","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_dm256| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|74.2 MB| ## References - https://huggingface.co/google/t5-efficient-small-dm256 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: NER Model for German author: John Snow Labs name: xlm_roberta_large_token_classifier_conll03 date: 2021-12-25 tags: [german, xlm, roberta, ner, de, open_source] task: Named Entity Recognition language: de edition: Spark NLP 3.3.4 spark_version: 2.4 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` and it's been fine-tuned for the German language, leveraging `XLM-RoBERTa` embeddings and `XlmRobertaForTokenClassification` for NER purposes. ## Predicted Entities `PER`, `ORG`, `LOC`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_conll03_de_3.3.4_2.4_1640443988652.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_conll03_de_3.3.4_2.4_1640443988652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Ibser |PER | |ASK Ebreichsdorf |ORG | |Admira Wacker Mödling |ORG | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_large_token_classifier_conll03| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|de| |Size:|1.8 GB| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/xlm-roberta-large-finetuned-conll03-german](https://huggingface.co/xlm-roberta-large-finetuned-conll03-german) --- layout: model title: Legal Standstill Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_standstill_agreement_bert date: 2023-01-29 tags: [en, legal, classification, standstill, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_standstill_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `standstill-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `standstill-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_standstill_agreement_bert_en_1.0.0_3.0_1674990206494.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_standstill_agreement_bert_en_1.0.0_3.0_1674990206494.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_standstill_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[standstill-agreement]| |[other]| |[other]| |[standstill-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_standstill_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.91 0.97 0.94 98 standstill-agreement 0.93 0.82 0.87 51 accuracy - - 0.92 149 macro-avg 0.92 0.90 0.91 149 weighted-avg 0.92 0.92 0.92 149 ``` --- layout: model title: Ganda asr_wav2vec2_xlsr_multilingual_56 TFWav2Vec2ForCTC from voidful author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_multilingual_56 date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_multilingual_56` is a Ganda model originally trained by voidful. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_multilingual_56_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_multilingual_56_lg_4.2.0_3.0_1664035886266.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_multilingual_56_lg_4.2.0_3.0_1664035886266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_multilingual_56', lang = 'lg') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_multilingual_56", lang = "lg") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_multilingual_56| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|lg| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering model (from tabo) author: John Snow Labs name: distilbert_qa_checkpoint_500_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `checkpoint-500-finetuned-squad` is a English model originally trained by `tabo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_checkpoint_500_finetuned_squad_en_4.0.0_3.0_1654723402338.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_checkpoint_500_finetuned_squad_en_4.0.0_3.0_1654723402338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_checkpoint_500_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_checkpoint_500_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_checkpoint_500_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tabo/checkpoint-500-finetuned-squad --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from davidenam) author: John Snow Labs name: xlmroberta_ner_davinam_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `davidenam`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_davinam_base_finetuned_panx_de_4.1.0_3.0_1660432159326.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_davinam_base_finetuned_panx_de_4.1.0_3.0_1660432159326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_davinam_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_davinam_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_davinam_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/davidenam/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English image_classifier_vit_modeversion1_m7_e4 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_modeversion1_m7_e4 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_modeversion1_m7_e4` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion1_m7_e4_en_4.1.0_3.0_1660171370244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_modeversion1_m7_e4_en_4.1.0_3.0_1660171370244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_modeversion1_m7_e4", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_modeversion1_m7_e4", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_modeversion1_m7_e4| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368_de_4.2.0_3.0_1664117272348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368_de_4.2.0_3.0_1664117272348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s368| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering (from avioo1) author: John Snow Labs name: roberta_qa_avioo1_roberta_base_squad2_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `avioo1`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_avioo1_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735375943.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_avioo1_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735375943.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_avioo1_roberta_base_squad2_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_avioo1_roberta_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_avioo1").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_avioo1_roberta_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/avioo1/roberta-base-squad2-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-32-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10_en_4.0.0_3.0_1655732483088.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10_en_4.0.0_3.0_1655732483088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_32d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|417.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-32-finetuned-squad-seed-10 --- layout: model title: Recognize Entities DL Pipeline for Polish - Medium author: John Snow Labs name: entity_recognizer_md date: 2021-03-22 tags: [open_source, polish, entity_recognizer_md, pipeline, pl] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: pl edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_pl_3.0.0_3.0_1616450153520.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_pl_3.0.0_3.0_1616450153520.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_md', lang = 'pl') annotations = pipeline.fullAnnotate(""Witaj z John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_md", lang = "pl") val result = pipeline.fullAnnotate("Witaj z John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Witaj z John Snow Labs! ""] result_df = nlu.load('pl.ner.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Witaj z John Snow Labs! '] | ['Witaj z John Snow Labs!'] | ['Witaj', 'z', 'John', 'Snow', 'Labs!'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|pl| --- layout: model title: Chinese BertForTokenClassification Base Cased model (from ckiplab) author: John Snow Labs name: bert_token_classifier_base_chinese_ws date: 2022-11-30 tags: [zh, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-chinese-ws` is a Chinese model originally trained by `ckiplab`. ## Predicted Entities `B`, `I` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_chinese_ws_zh_4.2.4_3.0_1669814825671.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_chinese_ws_zh_4.2.4_3.0_1669814825671.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_chinese_ws","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_chinese_ws","zh") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_base_chinese_ws| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|zh| |Size:|381.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ckiplab/bert-base-chinese-ws - https://github.com/ckiplab/ckip-transformers - https://muyang.pro - https://ckip.iis.sinica.edu.tw - https://github.com/ckiplab/ckip-transformers - https://github.com/ckiplab/ckip-transformers --- layout: model title: English BertForMaskedLM Cased model (from antoinelouis) author: John Snow Labs name: bert_embeddings_netbert date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `netbert` is a English model originally trained by `antoinelouis`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_netbert_en_4.2.4_3.0_1670022703984.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_netbert_en_4.2.4_3.0_1670022703984.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_netbert","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_netbert","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_netbert| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|405.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/antoinelouis/netbert - https://github.com/antoiloui/netbert/blob/master/docs/index.md --- layout: model title: Multilingual BertForQuestionAnswering model (from alon-albalak) author: John Snow Labs name: bert_qa_bert_base_multilingual_xquad date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-xquad` is a Multilingual model orginally trained by `alon-albalak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_xquad_xx_4.0.0_3.0_1654180335035.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_xquad_xx_4.0.0_3.0_1654180335035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_xquad","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_xquad","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.xquad.bert.multilingual_base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_xquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|626.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/alon-albalak/bert-base-multilingual-xquad - https://github.com/deepmind/xquad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from tusbaki) author: John Snow Labs name: distilbert_qa_tusbaki_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `tusbaki`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_tusbaki_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772936018.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_tusbaki_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772936018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tusbaki_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tusbaki_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_tusbaki_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tusbaki/distilbert-base-uncased-finetuned-squad --- layout: model title: English XlmRoBertaForQuestionAnswering (from Srini99) author: John Snow Labs name: xlm_roberta_qa_TQA date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `TQA` is a English model originally trained by `Srini99`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_TQA_en_4.0.0_3.0_1655983669052.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_TQA_en_4.0.0_3.0_1655983669052.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_TQA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_TQA","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.by_Srini99").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_TQA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|2.1 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Srini99/TQA --- layout: model title: POS Tagger Clinical author: John Snow Labs name: pos_clinical class: PerceptronModel language: en nav_key: models repository: clinical/models date: 2019-04-30 task: Part of Speech Tagging edition: Healthcare NLP 2.0.2 spark_version: 2.4 tags: [clinical, licensed, pos,en] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Sets a Part-Of-Speech tag to each word within a sentence. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/pos_clinical_en_2.0.2_2.4_1556660550177.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/pos_clinical_en_2.0.2_2.4_1556660550177.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_clinical","en","clinical/models")\ .setInputCols(["token","sentence"])\ .setOutputCol("pos") pos_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(pos_pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) result = light_pipeline.fullAnnotate("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.""") ``` ```scala val pos = PerceptronModel.pretrained("pos_clinical","en","clinical/models") .setInputCols("token","sentence") .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.pos.clinical").predict("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.""") ```

{:.h2_title} ## Results ```bash [Annotation(pos, 0, 1, NN, {'word': 'He'}), Annotation(pos, 3, 5, VBD, {'word': 'was'}), Annotation(pos, 7, 11, VVN, {'word': 'given'}), Annotation(pos, 13, 19, NNS, {'word': 'boluses'}), Annotation(pos, 21, 22, II, {'word': 'of'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | pos_clinical | | Type: | PerceptronModel | | Compatibility: | Spark NLP 2.0.2+ | | License: | Licensed | | Edition: | Official | |Input labels: | [token, sentence] | |Output labels: | [pos] | | Language: | en | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained with MedPost dataset. --- layout: model title: Detect problem, test, treatment in medical text (biobert) author: John Snow Labs name: ner_clinical_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect problem, test, treatment in medical text using pretrained NER model. ## Predicted Entities `PROBLEM`, `TREATMENT`, `TEST` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_biobert_en_3.0.0_3.0_1617260812919.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_biobert_en_3.0.0_3.0_1617260812919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------+---------+ |chunk |ner_label| +---------------------------------------------------+---------+ |congestion |PROBLEM | |some mild problems with his breathing while feeding|PROBLEM | |any perioral cyanosis |PROBLEM | |retractions |PROBLEM | |a tactile temperature |PROBLEM | |Tylenol |TREATMENT| |some decreased p.o |PROBLEM | |His normal breast-feeding |TEST | |his respiratory congestion |PROBLEM | |more tired |PROBLEM | |fussy |PROBLEM | |albuterol treatments |TREATMENT| |His urine output |TEST | |any diarrhea |PROBLEM | +---------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: Smaller BERT Sentence Embeddings (L-12_H-768_A-12) author: John Snow Labs name: sent_small_bert_L12_768 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L12_768_en_2.6.0_2.4_1598351662548.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L12_768_en_2.6.0_2.4_1598351662548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L12_768", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L12_768", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L12_768').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_small_bert_L12_768_embeddings 0 I hate cancer [0.128899946808815, -0.2827132046222687, 0.165... 1 Antibiotics aren't painkiller [-0.2957196533679962, -0.1430252194404602, 0.0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L12_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1 --- layout: model title: Google's Tapas Table Understanding (Base, WIKISQL) author: John Snow Labs name: table_qa_tapas_base_finetuned_wikisql_supervised date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Base Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_base_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530699686.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_base_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530699686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_base_finetuned_wikisql_supervised","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_512_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|432.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-4 --- layout: model title: Pipeline to Detect Mentions of Tumors in Text author: John Snow Labs name: nerdl_tumour_demo_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [nerdl_tumour_demo](https://nlp.johnsnowlabs.com/2021/04/01/nerdl_tumour_demo_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/nerdl_tumour_demo_pipeline_en_4.3.0_3.2_1678837464087.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/nerdl_tumour_demo_pipeline_en_4.3.0_3.2_1678837464087.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("nerdl_tumour_demo_pipeline", "en", "clinical/models") text = '''The final diagnosis was metastatic breast carcinoma, and it was classified as T2N1M1 stage IV. The histological grade of this 4 cm tumor was grade 2.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("nerdl_tumour_demo_pipeline", "en", "clinical/models") val text = "The final diagnosis was metastatic breast carcinoma, and it was classified as T2N1M1 stage IV. The histological grade of this 4 cm tumor was grade 2." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:-------------|:-------------| | 0 | breast carcinoma | 35 | 50 | Localization | | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_tumour_demo_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Detect clinical entities (ner_healthcare_slim) author: John Snow Labs name: ner_healthcare_slim date: 2021-04-01 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect clinical entities in German text using pretrained NER model ## Predicted Entities `TREATMENT`, `PERSON`, `BODY_PART`, `TIME_INFORMATION`, `MEDICAL_CONDITION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HEALTHCARE_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_slim_de_3.0.0_3.0_1617260856273.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_slim_de_3.0.0_3.0_1617260856273.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare_slim| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| --- layout: model title: Legal Health and Safety Clause Binary Classifier author: John Snow Labs name: legclf_health_and_safety_clause date: 2022-12-07 tags: [en, legal, classification, clauses, health_and_safety, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `health-and-safety` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `health-and-safety`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_health_and_safety_clause_en_1.0.0_3.0_1670445028648.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_health_and_safety_clause_en_1.0.0_3.0_1670445028648.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_health_and_safety_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[health-and-safety]| |[other]| |[other]| |[health-and-safety]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_health_and_safety_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support health-and-safety 1.00 0.85 0.92 27 other 0.95 1.00 0.97 73 accuracy - - 0.96 100 macro-avg 0.97 0.93 0.95 100 weighted-avg 0.96 0.96 0.96 100 ``` --- layout: model title: English AlbertForQuestionAnswering model (from mfeb) author: John Snow Labs name: albert_qa_xxlarge_v2_squad2 date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `albert-xxlarge-v2-squad2` is a English model originally trained by `mfeb`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_xxlarge_v2_squad2_en_4.0.0_3.0_1656064012371.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_xxlarge_v2_squad2_en_4.0.0_3.0_1656064012371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_xxlarge_v2_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_xxlarge_v2_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.albert.xxl_v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_xxlarge_v2_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|771.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mfeb/albert-xxlarge-v2-squad2 --- layout: model title: Detect Posology concepts (clinical_medium) author: John Snow Labs name: ner_posology_emb_clinical_medium date: 2023-04-12 tags: [ner, licensed, english, clinical, posology, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects Drug, Dosage, and administration instructions in text using pretrained NER model. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_POSOLOGY.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_emb_clinical_medium_en_4.3.2_3.0_1681315841950.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_emb_clinical_medium_en_4.3.2_3.0_1681315841950.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | begin | end | entities | |---:|:--------------|--------:|------:|:-----------| | 0 | Aspirin | 29 | 35 | DRUG | | 1 | 81 milligrams | 37 | 49 | STRENGTH | | 2 | QDay | 51 | 54 | FREQUENCY | | 3 | insulin | 57 | 63 | DRUG | | 4 | 50 units | 65 | 72 | STRENGTH | | 5 | HCTZ | 82 | 85 | DRUG | | 6 | 50 mg | 87 | 91 | STRENGTH | | 7 | QDay | 93 | 96 | FREQUENCY | | 8 | Nitroglycerin | 99 | 111 | DRUG | | 9 | 1/150 | 113 | 117 | STRENGTH | | 10 | sublingually | 119 | 130 | ROUTE | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.8 MB| ## Benchmarking ```bash precision recall f1-score support DRUG 0.91 0.91 0.91 2252 STRENGTH 0.88 0.93 0.91 2290 FREQUENCY 0.90 0.94 0.92 1782 DURATION 0.78 0.84 0.81 463 DOSAGE 0.66 0.63 0.65 476 ROUTE 0.89 0.89 0.89 394 FORM 0.86 0.76 0.81 773 micro avg 0.87 0.89 0.88 8430 macro avg 0.84 0.84 0.84 8430 weighted avg 0.87 0.89 0.88 8430 ``` --- layout: model title: Spanish Part of Speech Tagger (Base) author: John Snow Labs name: roberta_pos_roberta_base_bne_capitel_pos date: 2022-05-03 tags: [roberta, pos, part_of_speech, es, open_source] task: Part of Speech Tagging language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-base-bne-capitel-pos` is a Spanish model orginally trained by `PlanTL-GOB-ES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_base_bne_capitel_pos_es_3.4.2_3.0_1651591392282.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_base_bne_capitel_pos_es_3.4.2_3.0_1651591392282.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_roberta_base_bne_capitel_pos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|447.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne-capitel-pos - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://sites.google.com/view/capitel2020 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from venkateshdas) author: John Snow Labs name: roberta_qa_base_squad2_ta_qna_10e date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-ta-qna-roberta10e` is a English model originally trained by `venkateshdas`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad2_ta_qna_10e_en_4.3.0_3.0_1674219728754.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad2_ta_qna_10e_en_4.3.0_3.0_1674219728754.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad2_ta_qna_10e","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad2_ta_qna_10e","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_squad2_ta_qna_10e| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/venkateshdas/roberta-base-squad2-ta-qna-roberta10e --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_2_lr_2e_5_bs_32_ep_3 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-set_date_2-lr-2e-5-bs-32-ep-3` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_2_lr_2e_5_bs_32_ep_3_en_4.0.0_3.0_1657188406969.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_2_lr_2e_5_bs_32_ep_3_en_4.0.0_3.0_1657188406969.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_2_lr_2e_5_bs_32_ep_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_2_lr_2e_5_bs_32_ep_3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_2_lr_2e_5_bs_32_ep_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-set_date_2-lr-2e-5-bs-32-ep-3 --- layout: model title: Legal Consulting agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_consulting_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_consulting_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class consulting-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `consulting-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_consulting_agreement_en_1.0.0_3.0_1666620877422.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_consulting_agreement_en_1.0.0_3.0_1666620877422.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_consulting_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[consulting-agreement]| |[other]| |[other]| |[consulting-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_consulting_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support consulting-agreement 0.92 0.85 0.88 26 other 0.91 0.96 0.93 45 accuracy - - 0.92 71 macro-avg 0.92 0.90 0.91 71 weighted-avg 0.92 0.92 0.91 71 ``` --- layout: model title: Polish DistilBERT Embeddings author: John Snow Labs name: distilbert_embeddings_distilbert_base_pl_cased date: 2022-04-12 tags: [distilbert, embeddings, pl, open_source] task: Embeddings language: pl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-pl-cased` is a Polish model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_pl_cased_pl_3.4.2_3.0_1649783929723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_pl_cased_pl_3.4.2_3.0_1649783929723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_pl_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pl| |Size:|225.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-pl-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English image_classifier_vit_ak__base_patch16_224_in21k_image_classification ViTForImageClassification from amitkayal author: John Snow Labs name: image_classifier_vit_ak__base_patch16_224_in21k_image_classification date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_ak__base_patch16_224_in21k_image_classification` is a English model originally trained by amitkayal. ## Predicted Entities `1647932708266`, `1647153073068`, `1648120851131`, `1647323870424`, `1647324831223`, `1648529040870`, `1647667510657`, `1647838548516`, `1647491988625`, `1647068900550`, `1647241002660`, `1647155940977`, `1648444866188`, `1648287279256`, `1648360065352`, `1646891748795`, `1647175174130`, `1647667513009`, `1647503446037`, `1648616524252`, `1647241522127`, `1648454681973`, `1647581857611`, `1647233597062`, `1647933757599`, `1647420385320`, `1648361316938`, `1647856085470`, `1647243922705`, `1647497162634`, `1647237935761`, `1648369037326`, `1648115339027`, `1647153746047`, `1648273037858`, `1647150662937`, `1646893741640`, `1647845708343`, `1647147746930`, `1648366090438`, `1647156193650`, `1648537457250`, `1647149733777`, `1648443306030`, `1648646879260`, `1648001685069`, `1648528121469`, `1647156345180`, `1648456544611`, `1648107120561`, `1648359826755`, `1648366661601`, `1647666899143`, `1647935446369`, `1647668429439`, `1647936918167`, `1647235612241`, `1648041520955`, `1647243637048`, `1647680921307`, `1647081327122`, `1647087753595`, `1648528673166`, `1648710643516`, `1647945116578`, `1647846670493`, `1648536302434`, `1647761641671`, `1647325936506`, `1647325395017`, `1647234311073`, `1647759532201`, `1647241685563`, `1647935761690`, `1647846942176`, `1648698605364`, `1647933173332`, `1647420602021`, `1647159771571`, `1647324549954`, `1647065648037`, `1648536755030`, `1647924696448`, `1647927510285`, `1646892894926`, `1647580734898`, `1648287633901`, `1648442962568`, `1648368434262`, `1646988520214`, `1648279394220`, `1647150432271`, `1648643933549`, `1647448253535`, `1647929786244`, `1648370352609`, `1647330838532`, `1647147396410`, `1648644032811`, `1647140664832`, `1648536664835`, `1647410160165`, `1647164611497`, `1648183419560`, `1647773005511`, `1646034720737`, `1647328253213`, `1647155473555`, `1647953067595`, `1648538213890`, `1647409255750`, `1647682028193`, `1648116419206`, `1647329803500`, `1647154529007`, `1648099843046`, `1647248948963`, `1648279061829`, `1648296194026`, `1648108046332`, `1648113127522`, `1648455583722`, `1647761642926`, `1648533677853`, `1647940472293`, `1648701651498`, `1648456403645`, `1647752727972`, `1647494398054`, `1647674319450`, `1646887547179`, `1647158162746`, `1647176402807`, `1647065469305`, `1647838279102`, `1647674991869`, `1648113569157`, `1647067760203`, `1648365815895`, `1647330963213`, `1647405478288`, `1648372401817`, `1648103720352`, `1648115162538`, `1647784242846`, `1647402768175`, `1647490410305`, `1648286780106`, `1648625933250`, `1648534124563`, `1647173945909`, `1647235889634`, `1648525350277`, `1647236596892`, `1648292928751`, `1648706018510`, `1648024698508`, `1648707239343`, `1647767885862`, `1647240848064`, `1648280849373`, `1648406457408`, `1647236473208`, `1647157232763`, `1647147535719`, `1648706506117`, `1648706245642`, `1647933556772`, `1648616269757`, `1646809025517`, `1647420348512`, `1647399162926`, `1647843749559`, `1647242632376`, `1648182014371`, `1647321722450`, `1648369375824`, `1647331957791`, `1647494265176`, `1647252866283`, `1647930894634`, `1646809256434`, `1648537084774`, `1648450201448`, `1646885340637`, `1647760733996`, `1648287529293` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ak__base_patch16_224_in21k_image_classification_en_4.1.0_3.0_1660167449128.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_ak__base_patch16_224_in21k_image_classification_en_4.1.0_3.0_1660167449128.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_ak__base_patch16_224_in21k_image_classification", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_ak__base_patch16_224_in21k_image_classification", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_ak__base_patch16_224_in21k_image_classification| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.4 MB| --- layout: model title: Part of Speech for Slovak author: John Snow Labs name: pos_ud_snk date: 2020-05-04 23:32:00 +0800 task: Part of Speech Tagging language: sk edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, sk] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_snk_sk_2.5.0_2.4_1588622627281.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_snk_sk_2.5.0_2.4_1588622627281.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_snk", "sk") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_snk", "sk") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny."""] pos_df = nlu.load('sk.pos.ud_snk').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='ADP', metadata={'word': 'Okrem'}), Row(annotatorType='pos', begin=6, end=9, result='DET', metadata={'word': 'toho'}), Row(annotatorType='pos', begin=10, end=10, result='PUNCT', metadata={'word': ','}), Row(annotatorType='pos', begin=12, end=13, result='SCONJ', metadata={'word': 'že'}), Row(annotatorType='pos', begin=15, end=16, result='AUX', metadata={'word': 'je'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_snk| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|sk| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_300000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-300000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_300000_cased_generator_de_3.4.4_3.0_1652786364457.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_300000_cased_generator_de_3.4.4_3.0_1652786364457.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_300000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-300000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6_en_4.0.0_3.0_1655731375791.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6_en_4.0.0_3.0_1655731375791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_128d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|422.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-6 --- layout: model title: French CamemBert Embeddings (from devtrent) author: John Snow Labs name: camembert_embeddings_devtrent_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `devtrent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_devtrent_generic_model_fr_3.4.4_3.0_1653987835010.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_devtrent_generic_model_fr_3.4.4_3.0_1653987835010.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_devtrent_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/devtrent/dummy-model --- layout: model title: Recognize Entities DL pipeline for German - Large author: John Snow Labs name: entity_recognizer_lg date: 2021-03-23 tags: [open_source, german, entity_recognizer_lg, pipeline, de] supported: true task: [Named Entity Recognition, Lemmatization] language: de edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_de_3.0.0_3.0_1616491780964.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_de_3.0.0_3.0_1616491780964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_lg', lang = 'de') annotations = pipeline.fullAnnotate(""Hallo aus John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_lg", lang = "de") val result = pipeline.fullAnnotate("Hallo aus John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hallo aus John Snow Labs! ""] result_df = nlu.load('de.ner.recognizer.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-------------------------------|:------------------------------|:------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hallo aus John Snow Labs! '] | ['Hallo aus John Snow Labs!'] | ['Hallo', 'aus', 'John', 'Snow', 'Labs!'] | [[-0.245989993214607,.,...]] | ['O', 'O', 'I-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|de| --- layout: model title: Relation extraction between dates and clinical entities author: John Snow Labs name: re_date_clinical date: 2021-01-18 task: Relation Extraction language: en nav_key: models edition: Spark NLP for Healthcare 2.7.1 spark_version: 2.4 tags: [en, relation_extraction, clinical, licensed] supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between date and related other entities. `1` : Shows there is a relation between the date entity and other clinical entities, `0` : Shows there is no relation between the date entity and other clinical entities. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_date_clinical_en_2.7.1_2.4_1611000334654.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_date_clinical_en_2.7.1_2.4_1611000334654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `re_date_clinical` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:----------------:|:--------------:|:---------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | re_date_clinical | 0,1 | ner_jsl | [“date-admission_discharge”,
“admission_discharge-date”,
“date-alcohol”,
“alcohol-date”,
“date-allergen”,
“allergen-date”,
“date-bmi”,
“bmi-date”,
“date-birth_entity”,
“birth_entity-date”,
“date-blood_pressure”,
“blood_pressure-date”,
“date-cerebrovascular_disease”,
“cerebrovascular_disease-date”,
“date-clinical_dept”,
“clinical_dept-date”,
“date-communicable_disease”,
“communicable_disease-date”,
“date-death_entity”,
“death_entity-date”,
“date-diabetes”,
“diabetes-date”,
“date-diet”,
“diet-date”,
“date-disease_syndrome_disorder”,
“disease_syndrome_disorder-date”,
“date-drug_brandname”,
“drug_brandname-date”,
“date-drug_ingredient”,
“drug_ingredient-date”,
“date-ekg_findings”,
“ekg_findings-date”,
“date-external_body_part_or_region”,
“external_body_part_or_region-date”,
“date-fetus_newborn”,
“fetus_newborn-date”,
“date-hdl”,
“hdl-date”,
“date-heart_disease”,
“heart_disease-date”,
“date-height”,
“height-date”,
“date-hyperlipidemia”,
“hyperlipidemia-date”,
“date-hypertension”,
“hypertension-date”,
“date-imagingfindings”,
“imagingfindings-date”,
“date-imaging_technique”,
“imaging_technique-date”,
“date-injury_or_poisoning”,
“injury_or_poisoning-date”,
“date-internal_organ_or_component”,
“internal_organ_or_component-date”,
“date-kidney_disease”,
“kidney_disease-date”,
“date-ldl”,
“ldl-date”,
“date-modifier”,
“modifier-date”,
“date-o2_saturation”,
“o2_saturation-date”,
“date-obesity”,
“obesity-date”,
“date-oncological”,
“oncological-date”,
“date-overweight”,
“overweight-date”,
“date-oxygen_therapy”,
“oxygen_therapy-date”,
“date-pregnancy”,
“pregnancy-date”,
“date-procedure”,
“procedure-date”,
“date-psychological_condition”,
“psychological_condition-date”,
“date-pulse”,
“pulse-date”,
“date-respiration”,
“respiration-date”,
“date-smoking”,
“smoking-date”,
“date-substance”,
“substance-date”,
“date-substance_quantity”,
“substance_quantity-date”,
“date-symptom”,
“symptom-date”,
“date-temperature”,
“temperature-date”,
“date-test”,
“test-date”,
“date-test_result”,
“test_result-date”,
“date-total_cholesterol”,
“total_cholesterol-date”,
“date-treatment”,
“treatment-date”,
“date-triglycerides”,
“triglycerides-date”,
“date-vs_finding”,
“vs_finding-date”,
“date-vaccine”,
“vaccine-date”,
“date-vital_signs_header”,
“vital_signs_header-date”,
“date-weight”,
“weight-date”,
“time-admission_discharge”,
“admission_discharge-time”,
“time-alcohol”,
“alcohol-time”,
“time-allergen”,
“allergen-time”,
“time-bmi”,
“bmi-time”,
“time-birth_entity”,
“birth_entity-time”,
“time-blood_pressure”,
“blood_pressure-time”,
“time-cerebrovascular_disease”,
“cerebrovascular_disease-time”,
“time-clinical_dept”,
“clinical_dept-time”,
“time-communicable_disease”,
“communicable_disease-time”,
“time-death_entity”,
“death_entity-time”,
“time-diabetes”,
“diabetes-time”,
“time-diet”,
“diet-time”,
“time-disease_syndrome_disorder”,
“disease_syndrome_disorder-time”,
“time-drug_brandname”,
“drug_brandname-time”,
“time-drug_ingredient”,
“drug_ingredient-time”,
“time-ekg_findings”,
“ekg_findings-time”,
“time-external_body_part_or_region”,
“external_body_part_or_region-time”,
“time-fetus_newborn”,
“fetus_newborn-time”,
“time-hdl”,
“hdl-time”,
“time-heart_disease”,
“heart_disease-time”,
“time-height”,
“height-time”,
“time-hyperlipidemia”,
“hyperlipidemia-time”,
“time-hypertension”,
“hypertension-time”,
“time-imagingfindings”,
“imagingfindings-time”,
“time-imaging_technique”,
“imaging_technique-time”,
“time-injury_or_poisoning”,
“injury_or_poisoning-time”,
“time-internal_organ_or_component”,
“internal_organ_or_component-time”,
“time-kidney_disease”,
“kidney_disease-time”,
“time-ldl”,
“ldl-time”,
“time-modifier”,
“modifier-time”,
“time-o2_saturation”,
“o2_saturation-time”,
“time-obesity”,
“obesity-time”,
“time-oncological”,
“oncological-time”,
“time-overweight”,
“overweight-time”,
“time-oxygen_therapy”,
“oxygen_therapy-time”,
“time-pregnancy”,
“pregnancy-time”,
“time-procedure”,
“procedure-time”,
“time-psychological_condition”,
“psychological_condition-time”,
“time-pulse”,
“pulse-time”,
“time-respiration”,
“respiration-time”,
“time-smoking”,
“smoking-time”,
“time-substance”,
“substance-time”,
“time-substance_quantity”,
“substance_quantity-time”,
“time-symptom”,
“symptom-time”,
“time-temperature”,
“temperature-time”,
“time-test”,
“test-time”,
“time-test_result”,
“test_result-time”,
“time-total_cholesterol”,
“total_cholesterol-time”,
“time-treatment”,
“treatment-time”,
“time-triglycerides”,
“triglycerides-time”,
“time-vs_finding”,
“vs_finding-time”,
“time-vaccine”,
“vaccine-time”,
“time-vital_signs_header”,
“vital_signs_header-time”,
“time-weight”,
“weight-time”,
“relativedate-admission_discharge”,
“admission_discharge-relativedate”,
“relativedate-alcohol”,
“alcohol-relativedate”,
“relativedate-allergen”,
“allergen-relativedate”,
“relativedate-bmi”,
“bmi-relativedate”,
“relativedate-birth_entity”,
“birth_entity-relativedate”,
“relativedate-blood_pressure”,
“blood_pressure-relativedate”,
“relativedate-cerebrovascular_disease”,
“cerebrovascular_disease-relativedate”,
“relativedate-clinical_dept”,
“clinical_dept-relativedate”,
“relativedate-communicable_disease”,
“communicable_disease-relativedate”,
“relativedate-death_entity”,
“death_entity-relativedate”,
“relativedate-diabetes”,
“diabetes-relativedate”,
“relativedate-diet”,
“diet-relativedate”,
“relativedate-disease_syndrome_disorder”,
“disease_syndrome_disorder-relativedate”,
“relativedate-drug_brandname”,
“drug_brandname-relativedate”,
“relativedate-drug_ingredient”,
“drug_ingredient-relativedate”,
“relativedate-ekg_findings”,
“ekg_findings-relativedate”,
“relativedate-external_body_part_or_region”,
“external_body_part_or_region-relativedate”,
“relativedate-fetus_newborn”,
“fetus_newborn-relativedate”,
“relativedate-hdl”,
“hdl-relativedate”,
“relativedate-heart_disease”,
“heart_disease-relativedate”,
“relativedate-height”,
“height-relativedate”,
“relativedate-hyperlipidemia”,
“hyperlipidemia-relativedate”,
“relativedate-hypertension”,
“hypertension-relativedate”,
“relativedate-imagingfindings”,
“imagingfindings-relativedate”,
“relativedate-imaging_technique”,
“imaging_technique-relativedate”,
“relativedate-injury_or_poisoning”,
“injury_or_poisoning-relativedate”,
“relativedate-internal_organ_or_component”,
“internal_organ_or_component-relativedate”,
“relativedate-kidney_disease”,
“kidney_disease-relativedate”,
“relativedate-ldl”,
“ldl-relativedate”,
“relativedate-modifier”,
“modifier-relativedate”,
“relativedate-o2_saturation”,
“o2_saturation-relativedate”,
“relativedate-obesity”,
“obesity-relativedate”,
“relativedate-oncological”,
“oncological-relativedate”,
“relativedate-overweight”,
“overweight-relativedate”,
“relativedate-oxygen_therapy”,
“oxygen_therapy-relativedate”,
“relativedate-pregnancy”,
“pregnancy-relativedate”,
“relativedate-procedure”,
“procedure-relativedate”,
“relativedate-psychological_condition”,
“psychological_condition-relativedate”,
“relativedate-pulse”,
“pulse-relativedate”,
“relativedate-respiration”,
“respiration-relativedate”,
“relativedate-smoking”,
“smoking-relativedate”,
“relativedate-substance”,
“substance-relativedate”,
“relativedate-substance_quantity”,
“substance_quantity-relativedate”,
“relativedate-symptom”,
“symptom-relativedate”,
“relativedate-temperature”,
“temperature-relativedate”,
“relativedate-test”,
“test-relativedate”,
“relativedate-test_result”,
“test_result-relativedate”,
“relativedate-total_cholesterol”,
“total_cholesterol-relativedate”,
“relativedate-treatment”,
“treatment-relativedate”,
“relativedate-triglycerides”,
“triglycerides-relativedate”,
“relativedate-vs_finding”,
“vs_finding-relativedate”,
“relativedate-vaccine”,
“vaccine-relativedate”,
“relativedate-vital_signs_header”,
“vital_signs_header-relativedate”,
“relativedate-weight”,
“weight-relativedate”,
“relativetime-admission_discharge”,
“admission_discharge-relativetime”,
“relativetime-alcohol”,
“alcohol-relativetime”,
“relativetime-allergen”,
“allergen-relativetime”,
“relativetime-bmi”,
“bmi-relativetime”,
“relativetime-birth_entity”,
“birth_entity-relativetime”,
“relativetime-blood_pressure”,
“blood_pressure-relativetime”,
“relativetime-cerebrovascular_disease”,
“cerebrovascular_disease-relativetime”,
“relativetime-clinical_dept”,
“clinical_dept-relativetime”,
“relativetime-communicable_disease”,
“communicable_disease-relativetime”,
“relativetime-death_entity”,
“death_entity-relativetime”,
“relativetime-diabetes”,
“diabetes-relativetime”,
“relativetime-diet”,
“diet-relativetime”,
“relativetime-disease_syndrome_disorder”,
“disease_syndrome_disorder-relativetime”,
“relativetime-drug_brandname”,
“drug_brandname-relativetime”,
“relativetime-drug_ingredient”,
“drug_ingredient-relativetime”,
“relativetime-ekg_findings”,
“ekg_findings-relativetime”,
“relativetime-external_body_part_or_region”,
“external_body_part_or_region-relativetime”,
“relativetime-fetus_newborn”,
“fetus_newborn-relativetime”,
“relativetime-hdl”,
“hdl-relativetime”,
“relativetime-heart_disease”,
“heart_disease-relativetime”,
“relativetime-height”,
“height-relativetime”,
“relativetime-hyperlipidemia”,
“hyperlipidemia-relativetime”,
“relativetime-hypertension”,
“hypertension-relativetime”,
“relativetime-imagingfindings”,
“imagingfindings-relativetime”,
“relativetime-imaging_technique”,
“imaging_technique-relativetime”,
“relativetime-injury_or_poisoning”,
“injury_or_poisoning-relativetime”,
“relativetime-internal_organ_or_component”,
“internal_organ_or_component-relativetime”,
“relativetime-kidney_disease”,
“kidney_disease-relativetime”,
“relativetime-ldl”,
“ldl-relativetime”,
“relativetime-modifier”,
“modifier-relativetime”,
“relativetime-o2_saturation”,
“o2_saturation-relativetime”,
“relativetime-obesity”,
“obesity-relativetime”,
“relativetime-oncological”,
“oncological-relativetime”,
“relativetime-overweight”,
“overweight-relativetime”,
“relativetime-oxygen_therapy”,
“oxygen_therapy-relativetime”,
“relativetime-pregnancy”,
“pregnancy-relativetime”,
“relativetime-procedure”,
“procedure-relativetime”,
“relativetime-psychological_condition”,
“psychological_condition-relativetime”,
“relativetime-pulse”,
“pulse-relativetime”,
“relativetime-respiration”,
“respiration-relativetime”,
“relativetime-smoking”,
“smoking-relativetime”,
“relativetime-substance”,
“substance-relativetime”,
“relativetime-substance_quantity”,
“substance_quantity-relativetime”,
“relativetime-symptom”,
“symptom-relativetime”,
“relativetime-temperature”,
“temperature-relativetime”,
“relativetime-test”,
“test-relativetime”,
“relativetime-test_result”,
“test_result-relativetime”,
“relativetime-total_cholesterol”,
“total_cholesterol-relativetime”,
“relativetime-treatment”,
“treatment-relativetime”,
“relativetime-triglycerides”,
“triglycerides-relativetime”,
“relativetime-vs_finding”,
“vs_finding-relativetime”,
“relativetime-vaccine”,
“vaccine-relativetime”,
“relativetime-vital_signs_header”,
“vital_signs_header-relativetime”,
“relativetime-weight”,
“weight-relativetime”] | Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, PerceptronModel, DependencyParserModel, WordEmbeddingsModel, NerDLModel, NerConverter, RelationExtractionModel.

## Results ```bash | | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |---|-----------|---------|---------------|-------------|------------------------------------------|---------|-------------|-------------|---------|------------| | 0 | 1 | Test | 24 | 25 | CT | Date | 31 | 37 | 1/12/95 | 1.0 | | 1 | 1 | Symptom | 45 | 84 | progressive memory and cognitive decline | Date | 92 | 98 | 8/11/94 | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_date_clinical| |Type:|re| |Compatibility:|Spark NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on data gathered and manually annotated by John Snow Labs ## Benchmarking ```bash label recall precision f1 0 0.74 0.71 0.72 1 0.94 0.95 0.94 ``` --- layout: model title: English image_classifier_vit_rare_bottle ViTForImageClassification from tmoodley author: John Snow Labs name: image_classifier_vit_rare_bottle date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rare_bottle` is a English model originally trained by tmoodley. ## Predicted Entities `Don Julio`, `bacardi`, `Jack Daniels`, `johnny walker`, `Southern Comfort` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_bottle_en_4.1.0_3.0_1660172098511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_bottle_en_4.1.0_3.0_1660172098511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rare_bottle", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rare_bottle", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rare_bottle| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: German BERT Base Cased Model author: John Snow Labs name: bert_base_german_cased date: 2021-05-20 tags: [open_source, embeddings, bert, german, de] task: Embeddings language: de edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The source data for the model consists of a recent Wikipedia dump, EU Bookshop corpus, Open Subtitles, CommonCrawl, ParaCrawl and News Crawl. This results in a dataset with a size of 16GB and 2,350,234,427 tokens. The model is trained with an initial sequence length of 512 subwords and was performed for 1.5M steps. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_german_cased_de_3.1.0_3.0_1621502949396.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_german_cased_de_3.1.0_3.0_1621502949396.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_german_cased", "de") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_german_cased", "de") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("de.embed.bert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_base_german_cased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|de| |Case sensitive:|true| ## Data Source https://huggingface.co/dbmdz/bert-base-german-cased ## Benchmarking ```bash For results on downstream tasks like NER or PoS tagging, please refer to [this repository](https://github.com/stefan-it/fine-tuned-berts-seq). ``` --- layout: model title: English ElectraForQuestionAnswering model (from Andranik) author: John Snow Labs name: electra_qa_TestQA2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `TestQA2` is a English model originally trained by `Andranik`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_TestQA2_en_4.0.0_3.0_1655920070692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_TestQA2_en_4.0.0_3.0_1655920070692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_TestQA2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_TestQA2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.electra").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_TestQA2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Andranik/TestQA2 --- layout: model title: Hindi DistilBERT Embeddings (from neuralspace-reverie) author: John Snow Labs name: distilbert_embeddings_indic_transformers_hi_distilbert date: 2022-04-12 tags: [distilbert, embeddings, hi, open_source] task: Embeddings language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indic-transformers-hi-distilbert` is a Hindi model orginally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_indic_transformers_hi_distilbert_hi_3.4.2_3.0_1649783440506.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_indic_transformers_hi_distilbert_hi_3.4.2_3.0_1649783440506.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_indic_transformers_hi_distilbert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|hi| |Size:|247.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-hi-distilbert - https://oscar-corpus.com/ --- layout: model title: English RobertaForQuestionAnswering (from nlpunibo) author: John Snow Labs name: roberta_qa_nlpunibo_roberta date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_nlpunibo_roberta_en_4.0.0_3.0_1655729625379.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_nlpunibo_roberta_en_4.0.0_3.0_1655729625379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_nlpunibo_roberta","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_nlpunibo_roberta","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.by_nlpunibo").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_nlpunibo_roberta| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/roberta --- layout: model title: Vietnamese T5ForConditionalGeneration Small Cased model (from NlpHUST) author: John Snow Labs name: t5_vi_small date: 2023-01-31 tags: [vi, open_source, t5, tensorflow] task: Text Generation language: vi edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-vi-en-small` is a Vietnamese model originally trained by `NlpHUST`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_vi_small_vi_4.3.0_3.0_1675156710542.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_vi_small_vi_4.3.0_3.0_1675156710542.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_vi_small","vi") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_vi_small","vi") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_vi_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|vi| |Size:|819.8 MB| ## References - https://huggingface.co/NlpHUST/t5-vi-en-small --- layout: model title: English DistilBertForTokenClassification Cased model (from f2io) author: John Snow Labs name: distilbert_token_classifier_ner_roles_openapi date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ner-roles-openapi` is a English model originally trained by `f2io`. ## Predicted Entities ``, `LOC`, `OR`, `PRG`, `ROLE`, `ORG`, `PER`, `ENTITY`, `MISC`, `ACTION` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.1_3.0_1678133755717.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.1_3.0_1678133755717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_ner_roles_openapi| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/f2io/ner-roles-openapi --- layout: model title: Translate English to Umbundu Pipeline author: John Snow Labs name: translate_en_umb date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, umb, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `umb` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_umb_xx_2.7.0_2.4_1609689689023.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_umb_xx_2.7.0_2.4_1609689689023.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_umb", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_umb", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.umb').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_umb| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el48 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el48` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el48_en_4.3.0_3.0_1675120232514.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el48_en_4.3.0_3.0_1675120232514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el48","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el48","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el48| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|401.7 MB| ## References - https://huggingface.co/google/t5-efficient-small-el48 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English asr_models_6 TFWav2Vec2ForCTC from niclas author: John Snow Labs name: asr_models_6 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_models_6` is a English model originally trained by niclas. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_models_6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_models_6_en_4.2.0_3.0_1664098721737.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_models_6_en_4.2.0_3.0_1664098721737.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_models_6", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_models_6", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_models_6| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Catalan Named Entity Recognition from project-aina author: cayorodriguez name: ner_bsc date: 2022-07-07 tags: [ner, bsc, projecte_aina, ca, open_source] task: Named Entity Recognition language: ca edition: Spark NLP 3.4.4 spark_version: 3.0 supported: false recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Based on the huggingface model `projecte-aina/roberta-base-ca-cased-ner`, this model requires a specific tokenizer (look at the Python Examples section). ## Predicted Entities `PER`, `ORG`, `LOC`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/cayorodriguez/ner_bsc_ca_3.4.4_3.0_1657197794383.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/cayorodriguez/ner_bsc_ca_3.4.4_3.0_1657197794383.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') ex_list = ["aprox\.","pàg\.","p\.ex\.","gen\.","feb\.","abr\.","jul\.","set\.","oct\.","nov\.","dec\.","dr\.","dra\.","sr\.","sra\.","srta\.","núm\.","st\.","sta\.","pl\.","etc\.", "ex\."] ex_list_all = [] ex_list_all.extend(ex_list) ex_list_all.extend([x[0].upper() + x[1:] for x in ex_list]) ex_list_all.extend([x.upper() for x in ex_list]) tokenizer = Tokenizer() \ .setInputCols(['sentence']).setOutputCol('token')\ .setInfixPatterns(["(d|D)(els)","(d|D)(el)","(a|A)(ls)","(a|A)(l)","(p|P)(els)","(p|P)(el)",\ "([A-zÀ-ú_@]+)(-[A-zÀ-ú_@]+)",\ "(d'|D')([·A-zÀ-ú@_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+","(l'|L')([·A-zÀ-ú_]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|'|,)+", \ "(l'|l'|s'|s'|d'|d'|m'|m'|n'|n'|D'|D'|L'|L'|S'|S'|N'|N'|M'|M')([A-zÀ-ú_]+)",\ """([A-zÀ-ú·]+)(\.|,|\)|\?|!|;|\:|\"|”)(\.|,|\)|\?|!|;|\:|\"|”)+""",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|,|;|:|\?|,)+",\ "([A-zÀ-ú·]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|')+([0-9A-zÀ-ú_]+)",\ "([0-9A-zÀ-ú·]+)(\.|\"|;|:|!|\?|$|$|”|“|'|,|%)",\ "(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+([0-9]+)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)+",\ "(d'|D'|l'|L')([·A-zÀ-ú@_]+)('l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(\.|\"|;|:|!|\?|\-|$|$|”|“|,)", \ "([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)([\.|\"|;|:|!|\?|\-|$|$|”|“|,]+)"]) \ .setExceptions(ex_list_all).fit(data) tokenClassifier = RoBertaForTokenClassification \ .pretrained('ner_bsc', 'ca', '@cayorodriguez') \ .setInputCols(['token', 'document']) \ .setOutputCol('ner') \ .setCaseSensitive(True) \ .setMaxSentenceLength(512) ner_converter = NerConverter() \ .setInputCols(['document', 'token', 'ner']) \ .setOutputCol('entities') pipeline = Pipeline(stages=[ document_assembler, tokenizer, tokenClassifier, ner_converter ]) example = spark.createDataFrame([['El meu nom es Carlos i visc a Catalunya!']]).toDF("text") result = pipeline.fit(example).transform(example) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bsc| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Community| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ca| |Size:|445.9 MB| |Case sensitive:|true| |Max sentence length:|128| ## References projecte-aina/ancora-ca-ner @ huggingface --- layout: model title: Part of Speech for Russian author: John Snow Labs name: pos_ud_gsd date: 2021-03-08 tags: [part_of_speech, open_source, russian, pos_ud_gsd, ru] task: Part of Speech Tagging language: ru edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADP - NOUN - PROPN - CCONJ - PUNCT - PRON - VERB - DET - ADJ - SCONJ - AUX - PART - NUM - ADV - SYM - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ru_3.0.0_3.0_1615230203372.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_ru_3.0.0_3.0_1615230203372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_gsd", "ru") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Здравствуйте из Джона Снежных Лабораторий! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_gsd", "ru") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Здравствуйте из Джона Снежных Лабораторий! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Здравствуйте из Джона Снежных Лабораторий! ""] token_df = nlu.load('ru.pos').predict(text) token_df ```

## Results ```bash token pos 0 Здравствуйте NOUN 1 из ADP 2 Джона PROPN 3 Снежных ADJ 4 Лабораторий PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|ru| --- layout: model title: Finnish BertForMaskedLM Base Cased model (from TurkuNLP) author: John Snow Labs name: bert_embeddings_base_finnish_cased_v1 date: 2022-12-02 tags: [fi, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: fi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-finnish-cased-v1` is a Finnish model originally trained by `TurkuNLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_finnish_cased_v1_fi_4.2.4_3.0_1670017531921.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_finnish_cased_v1_fi_4.2.4_3.0_1670017531921.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_finnish_cased_v1","fi") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_finnish_cased_v1","fi") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_finnish_cased_v1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fi| |Size:|467.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1 - https://arxiv.org/abs/1912.07076 - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/master/multilingual.md - https://raw.githubusercontent.com/TurkuNLP/FinBERT/master/img/yle-ylilauta-curves.png - https://fasttext.cc/ - https://github.com/spyysalo/finbert-text-classification - https://github.com/spyysalo/yle-corpus - https://github.com/spyysalo/ylilauta-corpus - https://arxiv.org/abs/1908.04212 - https://github.com/Traubert/FiNer-rules - https://arxiv.org/pdf/1908.04212.pdf - https://github.com/jouniluoma/keras-bert-ner - https://github.com/mpsilfve/finer-data - https://universaldependencies.org/ - https://github.com/spyysalo/bert-pos - http://hdl.handle.net/11234/1-2837 - http://dl.turkunlp.org/finbert/bert-base-finnish-uncased.zip - http://dl.turkunlp.org/finbert/bert-base-finnish-cased.zip --- layout: model title: English DistilBertForQuestionAnswering Cased model (from aszidon) author: John Snow Labs name: distilbert_qa_custom3 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom3` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom3_en_4.3.0_3.0_1672774647602.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom3_en_4.3.0_3.0_1672774647602.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom3 --- layout: model title: Chinese Part of Speech Tagger(UPOS, Chinese Wikipedia Texts) author: John Snow Labs name: bert_pos_chinese_bert_wwm_ext_upos date: 2022-04-26 tags: [bert, pos, part_of_speech, zh, open_source] task: Part of Speech Tagging language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-bert-wwm-ext-upos` is a Chinese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_chinese_bert_wwm_ext_upos_zh_3.4.2_3.0_1650993237648.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_chinese_bert_wwm_ext_upos_zh_3.4.2_3.0_1650993237648.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_chinese_bert_wwm_ext_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|zh| |Size:|382.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/chinese-bert-wwm-ext-upos - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: French RobertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: roberta_qa_ADDI_FR_RoBERTa date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: fr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-FR-RoBERTa` is a French model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FR_RoBERTa_fr_4.0.0_3.0_1655726415021.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FR_RoBERTa_fr_4.0.0_3.0_1655726415021.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ADDI_FR_RoBERTa","fr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ADDI_FR_RoBERTa","fr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fr.answer_question.roberta.fr_tuned.by_Gantenbein").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ADDI_FR_RoBERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fr| |Size:|422.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-FR-RoBERTa --- layout: model title: Legal NER from Sigma Absa Dataset (PER+Pronouns) author: John Snow Labs name: legner_sigma_absa_people date: 2022-12-16 tags: [sigma, absa, people, pronouns, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Legal NER model trained on the Sigma Absa Dataset for legal sentiment analysis on legal parties, including correference pronouns (he, him, their...). This is the first component which extracts those people names and pronouns and NER. You have the second component, which does Assertion Status to retrieve sentiment, on `legassertion_sigma_absa_sentiment` ## Predicted Entities `PER`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_sigma_absa_people_en_1.0.0_3.0_1671202164090.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_sigma_absa_people_en_1.0.0_3.0_1671202164090.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner = legal.NerDLModel.pretrained("legner_sigma_absa_people", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("label") pipe = nlp.Pipeline(stages = [ document_assembler, sentence_detector, tokenizer, embeddings, ner]) text = "Petitioner Jae Lee moved to the United States from South Korea with his parents when he was 13." sdf = spark.createDataFrame([[text]]).toDF("text") res = pipe.fit(sdf).transform(sdf) import pyspark.sql.functions as F res.select(F.explode(F.arrays_zip(res.token.result, res.label.result, res.label.metadata)).alias("cols"))\ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ner_label"), F.expr("cols['2']['confidence']").alias("confidence")).show(200, truncate=100) ```

## Results ```bash +----------+---------+----------+ | token|ner_label|confidence| +----------+---------+----------+ |Petitioner| B-PER| 0.9997| | Jae| I-PER| 0.9952| | Lee| I-PER| 0.9951| | moved| O| 1.0| | to| O| 1.0| | the| O| 1.0| | United| O| 1.0| | States| O| 1.0| | from| O| 1.0| | South| O| 1.0| | Korea| O| 1.0| | with| O| 1.0| | his| B-PER| 1.0| | parents| O| 0.9998| | when| O| 1.0| | he| B-PER| 1.0| | was| O| 1.0| | 13| O| 1.0| | .| O| 1.0| +----------+---------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_sigma_absa_people| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.1 MB| ## References https://metatext.io/datasets/sigmalaw-absa ## Benchmarking ```bash label tp fp fn prec rec f1 I-PER 43 2 0 0.95555556 1.0 0.97727275 B-PER 777 11 15 0.9860406 0.9810606 0.98354435 Macro-average 820 13 15 0.9707981 0.9905303 0.9805649 Micro-average 820 13 15 0.9843938 0.98203593 0.9832135 ``` --- layout: model title: English Bert Embeddings (from beatrice-portelli) author: John Snow Labs name: bert_embeddings_DiLBERT date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `DiLBERT` is a English model orginally trained by `beatrice-portelli`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_DiLBERT_en_3.4.2_3.0_1649672580183.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_DiLBERT_en_3.4.2_3.0_1649672580183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_DiLBERT| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|410.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/beatrice-portelli/DiLBERT - https://github.com/KevinRoitero/dilbert --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl16_en_4.3.0_3.0_1675121587021.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl16_en_4.3.0_3.0_1675121587021.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|288.9 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Confidential Information Clause Binary Classifier author: John Snow Labs name: legclf_confidential_information_clause date: 2022-12-18 tags: [en, legal, classification, licensed, clause, bert, confidential, information, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the confidential-information clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it’s better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `confidential-information`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_confidential_information_clause_en_1.0.0_3.0_1671393646578.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_confidential_information_clause_en_1.0.0_3.0_1671393646578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_confidential_information_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[confidential-information]| |[other]| |[other]| |[confidential-information]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_confidential_information_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support confidential-information 1.00 1.00 1.00 25 other 1.00 1.00 1.00 39 accuracy - - 1.00 64 macro-avg 1.00 1.00 1.00 64 weighted-avg 1.00 1.00 1.00 64 ``` --- layout: model title: Vaccine Sentiment Classifier (PHS-BERT) author: John Snow Labs name: classifierdl_vaccine_sentiment date: 2022-07-28 tags: [public_health, en, licensed, vaccine_sentiment, classification] task: Sentiment Analysis language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [PHS-BERT](https://arxiv.org/abs/2204.04521) based sentimental analysis model that can extract information from COVID-19 Vaccine-related tweets. The model predicts whether a tweet contains positive, negative, or neutral sentiments about COVID-19 Vaccines. ## Predicted Entities `negative`, `positive`, `neutral` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_vaccine_sentiment_en_4.0.0_3.0_1658998378316.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_vaccine_sentiment_en_4.0.0_3.0_1658998378316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") bert_embeddings = BertEmbeddings.pretrained("bert_embeddings_phs_bert", "en", "public/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ embeddingsSentence = SentenceEmbeddings() \ .setInputCols(["sentence", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifierdl = ClassifierDLModel.pretrained('classifierdl_vaccine_sentiment', "en", "clinical/models")\ .setInputCols(['sentence', 'token', 'sentence_embeddings'])\ .setOutputCol('class') pipeline = Pipeline( stages = [ document_assembler, tokenizer, bert_embeddings, embeddingsSentence, classifierdl ]) text_list = ['A little bright light for an otherwise dark week. Thanks researchers, and frontline workers. Onwards.', 'People with a history of severe allergic reaction to any component of the vaccine should not take.', '43 million doses of vaccines administrated worldwide...Production capacity of CHINA to reach 4 b'] data = spark.createDataFrame(text_list, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_phs_bert", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val sentence_embeddings = SentenceEmbeddings() .setInputCols(Array("sentence", "embeddings")) .setOutputCol("sentence_embeddings") .setPoolingStrategy("AVERAGE") val classifier = ClassifierDLModel.pretrained("classifierdl_vaccine_sentiment", "en", "clinical/models") .setInputCols(Array("sentence", "token", "sentence_embeddings")) .setOutputCol("class") val bert_clf_pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, sentence_embeddings, classifier)) val data = Seq(Array("A little bright light for an otherwise dark week. Thanks researchers, and frontline workers. Onwards.", "People with a history of severe allergic reaction to any component of the vaccine should not take.", "43 million doses of vaccines administrated worldwide...Production capacity of CHINA to reach 4 b")).toDS.toDF("text") val result = bert_clf_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.vaccine_sentiment").predict("""A little bright light for an otherwise dark week. Thanks researchers, and frontline workers. Onwards.""") ```

## Results ```bash +-----------------------------------------------------------------------------------------------------+----------+ |text |class | +-----------------------------------------------------------------------------------------------------+----------+ |A little bright light for an otherwise dark week. Thanks researchers, and frontline workers. Onwards.|[positive]| |People with a history of severe allergic reaction to any component of the vaccine should not take. |[negative]| |43 million doses of vaccines administrated worldwide...Production capacity of CHINA to reach 4 b |[neutral] | +-----------------------------------------------------------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_vaccine_sentiment| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|24.2 MB| ## References Curated from several academic and in-house datasets. ## Benchmarking ```bash label precision recall f1-score support neutral 0.76 0.72 0.74 1008 positive 0.80 0.79 0.80 966 negative 0.76 0.81 0.78 916 accuracy - - 0.77 2890 macro-avg 0.77 0.77 0.77 2890 weighted-avg 0.77 0.77 0.77 2890 ``` --- layout: model title: Portuguese asr_bp_commonvoice10_xlsr TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_bp_commonvoice10_xlsr date: 2022-09-26 tags: [wav2vec2, pt, audio, open_source, asr] task: Automatic Speech Recognition language: pt edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bp_commonvoice10_xlsr` is a Portuguese model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bp_commonvoice10_xlsr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bp_commonvoice10_xlsr_pt_4.2.0_3.0_1664196471529.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bp_commonvoice10_xlsr_pt_4.2.0_3.0_1664196471529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bp_commonvoice10_xlsr", "pt")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bp_commonvoice10_xlsr", "pt") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bp_commonvoice10_xlsr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|pt| |Size:|756.4 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Sango author: John Snow Labs name: opus_mt_en_sg date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, sg, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `sg` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_sg_xx_2.7.0_2.4_1609163620103.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_sg_xx_2.7.0_2.4_1609163620103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_sg| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Arabic BertForQuestionAnswering model (from gfdgdfgdg) author: John Snow Labs name: bert_qa_arap_qa_bert_large_v2 date: 2022-06-02 tags: [ar, open_source, question_answering, bert] task: Question Answering language: ar edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `arap_qa_bert_large_v2` is a Arabic model orginally trained by `gfdgdfgdg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_arap_qa_bert_large_v2_ar_4.0.0_3.0_1654179148138.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_arap_qa_bert_large_v2_ar_4.0.0_3.0_1654179148138.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_arap_qa_bert_large_v2","ar") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_arap_qa_bert_large_v2","ar") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ar.answer_question.bert.large_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_arap_qa_bert_large_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ar| |Size:|1.4 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/gfdgdfgdg/arap_qa_bert_large_v2 --- layout: model title: English RobertaForQuestionAnswering Cased model (from Andranik) author: John Snow Labs name: roberta_qa_test_v1 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `TestQaV1` is a English model originally trained by `Andranik`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_test_v1_en_4.3.0_3.0_1674208905005.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_test_v1_en_4.3.0_3.0_1674208905005.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_test_v1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_test_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_test_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Andranik/TestQaV1 --- layout: model title: XLNet Base CoNLL-03 NER Pipeline author: John Snow Labs name: xlnet_base_token_classifier_conll03_pipeline date: 2022-04-21 tags: [ner, english, xlnet, base, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlnet_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/28/xlnet_base_token_classifier_conll03_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlnet_base_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650540715035.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlnet_base_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650540715035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlnet_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("xlnet_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xlsr_53_bemba_5hrs', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xlsr_53_bemba_5hrs", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xlsr_53_bemba_5hrs| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Stop Words Cleaner for Armenian author: John Snow Labs name: stopwords_hy date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: hy edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, hy] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_hy_hy_2.5.4_2.4_1594742439626.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_hy_hy_2.5.4_2.4_1594742439626.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_hy", "hy") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար:") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_hy", "hy") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար:").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար"""] stopword_df = nlu.load('hy.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=10, result='Հյուսիսային', metadata={'sentence': '0'}), Row(annotatorType='token', begin=12, end=18, result='թագավոր', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=27, result='լինելուց', metadata={'sentence': '0'}), Row(annotatorType='token', begin=29, end=32, result='բացի', metadata={'sentence': '0'}), Row(annotatorType='token', begin=33, end=33, result=',', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_hy| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|hy| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English asr_wav2vec2_base_timit_demo_colab1_by_sherry7144 TFWav2Vec2ForCTC from sherry7144 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab1_by_sherry7144 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab1_by_sherry7144` is a English model originally trained by sherry7144. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab1_by_sherry7144_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab1_by_sherry7144_en_4.2.0_3.0_1664018634076.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab1_by_sherry7144_en_4.2.0_3.0_1664018634076.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab1_by_sherry7144", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab1_by_sherry7144", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab1_by_sherry7144| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Legal Settlement Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_settlement_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, settlement, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_settlement_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `settlement-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `settlement-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_settlement_agreement_bert_en_1.0.0_3.0_1669371428132.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_settlement_agreement_bert_en_1.0.0_3.0_1669371428132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_settlement_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[settlement-agreement]| |[other]| |[other]| |[settlement-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_settlement_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.93 0.95 0.94 65 settlement-agreement 0.90 0.85 0.88 33 accuracy - - 0.92 98 macro-avg 0.91 0.90 0.91 98 weighted-avg 0.92 0.92 0.92 98 ``` --- layout: model title: Word2Vec Embeddings in English (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_en_3.4.1_3.0_1647368332088.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_en_3.4.1_3.0_1647368332088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_pubmedbykrs","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_pubmedbykrs","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_pubmedbykrs| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/pythonist/distilbert-base-uncased-finetuned-pubmedbykrs --- layout: model title: Google T5 for Closed Book Question Answering Small author: John Snow Labs name: google_t5_small_ssm_nq date: 2021-01-08 task: Question Answering language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, t5, seq2seq, question_answering, en] supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The model was pre-trained using T5's denoising objective on [C4](https://huggingface.co/datasets/c4), subsequently additionally pre-trained using REALM's salient span masking objective on Wikipedia, and finally fine-tuned on [Natural Questions (NQ)](https://huggingface.co/datasets/natural_questions). Note: The model was fine-tuned on 100% of the train splits of Natural Questions (NQ) for 10k steps. Other community Checkpoints: here Paper: [How Much Knowledge Can You Pack Into the Parameters of a Language Model?](https://arxiv.org/abs/1910.10683.pdf) {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/google_t5_small_ssm_nq_en_2.7.1_2.4_1610137175322.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/google_t5_small_ssm_nq_en_2.7.1_2.4_1610137175322.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Either set the following tasks or have them inline with your input: - nq question: - trivia question: - question: - nq:

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = T5Transformer() \ .pretrained("google_t5_small_ssm_nq") \ .setTask("nq:")\ .setMaxOutputLength(200)\ .setInputCols(["documents"]) \ .setOutputCol("answer") pipeline = Pipeline().setStages([document_assembler, t5]) results = pipeline.fit(data_df).transform(data_df) results.select("answer.result").show(truncate=False) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer .pretrained("google_t5_small_ssm_nq") .setTask("nq:") .setInputCols(Array("documents")) .setOutputCol("answer") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val model = pipeline.fit(dataDf) val results = model.transform(dataDf) results.select("answer.result").show(truncate = false) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|google_t5_small_ssm_nq| |Compatibility:|Spark NLP 2.7.1+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[t5]| |Language:|en| ## Data Source The model was pre-trained using T5's denoising objective on C4, subsequently additionally pre-trained using REALM's salient span masking objective on Wikipedia, and finally fine-tuned on Natural Questions (NQ). --- layout: model title: Malay ALBERT Embeddings (Base) author: John Snow Labs name: albert_embeddings_albert_base_bahasa_cased date: 2022-04-14 tags: [albert, embeddings, ms, open_source] task: Embeddings language: ms edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-base-bahasa-cased` is a Malay model orginally trained by `malay-huggingface`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_bahasa_cased_ms_3.4.2_3.0_1649954337875.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_bahasa_cased_ms_3.4.2_3.0_1649954337875.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_base_bahasa_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ms| |Size:|45.7 MB| |Case sensitive:|false| ## References - https://huggingface.co/malay-huggingface/albert-base-bahasa-cased - https://github.com/huseinzol05/malay-dataset/tree/master/dumping/clean - https://github.com/huseinzol05/malay-dataset/tree/master/corpus/pile - https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/albert --- layout: model title: Fast Neural Machine Translation Model from Finnish to English author: John Snow Labs name: opus_mt_fi_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, fi, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `fi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_fi_en_xx_2.7.0_2.4_1609170172454.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_fi_en_xx_2.7.0_2.4_1609170172454.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_fi_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from leixu) author: John Snow Labs name: xlmroberta_ner_leixu_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `leixu`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_leixu_base_finetuned_panx_de_4.1.0_3.0_1660435051026.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_leixu_base_finetuned_panx_de_4.1.0_3.0_1660435051026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_leixu_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_leixu_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_leixu_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/leixu/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English BertForQuestionAnswering model (from MichelBartels) author: John Snow Labs name: bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tinybert-6l-768d-squad2-large-teacher-finetuned` is a English model orginally trained by `MichelBartels`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_en_4.0.0_3.0_1654192477274.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned_en_4.0.0_3.0_1654192477274.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_tiny_768d.by_MichelBartels").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tinybert_6l_768d_squad2_large_teacher_finetuned| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|249.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MichelBartels/tinybert-6l-768d-squad2-large-teacher-finetuned --- layout: model title: XLNet Base CoNLL-03 NER Pipeline author: John Snow Labs name: xlnet_base_token_classifier_conll03_pipeline date: 2022-06-19 tags: [ner, english, xlnet, base, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlnet_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/28/xlnet_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlnet_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655654102760.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlnet_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655654102760.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_zls| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pre-trained Pipeline for Few-NERD NER Model author: John Snow Labs name: nerdl_fewnerd_subentity_100d_pipeline date: 2021-09-28 tags: [fewnerd, ner, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.3.0 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on Few-NERD/inter public dataset and it extracts 66 entities that are in general scope. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_pipeline_en_3.3.0_2.4_1632828937931.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_pipeline_en_3.3.0_2.4_1632828937931.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python fewnerd_pipeline = PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") fewnerd_pipeline.annotate("""12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.""") ``` ```scala val pipeline = new PretrainedPipeline("nerdl_fewnerd_subentity_100d_pipeline", lang = "en") val result = pipeline.fullAnnotate("12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.")(0) ```

## Results ```bash [Row(mappings=[Row(annotatorType='labeled_dependency', begin=46, end=49, result='AMZN', metadata={'sentence': '0', 'chunk': '0', 'entity': 'AMZN', 'relation': 'ticker', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=46, end=49, result='Amazon.com Inc.', metadata={'sentence': '0', 'chunk': '0', 'entity': 'AMZN', 'relation': 'company_name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=46, end=49, result='Amazon.com', metadata={'sentence': '0', 'chunk': '0', 'entity': 'AMZN', 'relation': 'short_name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=46, end=49, result='Retail - Apparel & Specialty', metadata={'sentence': '0', 'chunk': '0', 'entity': 'AMZN', 'relation': 'industry', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=46, end=49, result='Consumer Cyclical', metadata={'sentence': '0', 'chunk': '0', 'entity': 'AMZN', 'relation': 'sector', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=57, end=61, result='NONE', metadata={'sentence': '0', 'chunk': '1', 'entity': 'today'}, embeddings=[])])] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_nasdaq_ticker| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|210.3 KB| ## References https://data.world/johnsnowlabs/list-of-companies-in-nasdaq-exchanges --- layout: model title: Vietnamese BertForQuestionAnswering model (from tiennvcs) author: John Snow Labs name: bert_qa_bert_base_uncased_finetuned_vi_infovqa date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: vi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-vi-infovqa` is a Vietnamese model orginally trained by `tiennvcs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_vi_infovqa_vi_4.0.0_3.0_1654181242200.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_vi_infovqa_vi_4.0.0_3.0_1654181242200.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_finetuned_vi_infovqa","vi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_finetuned_vi_infovqa","vi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("vi.answer_question.bert.vi_infovqa.base_uncased.by_tiennvcs").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_finetuned_vi_infovqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|vi| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tiennvcs/bert-base-uncased-finetuned-vi-infovqa --- layout: model title: English asr_wolof_ASR TFWav2Vec2ForCTC from khady author: John Snow Labs name: pipeline_asr_wolof_ASR date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wolof_ASR` is a English model originally trained by khady. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wolof_ASR_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wolof_ASR_en_4.2.0_3.0_1664024690847.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wolof_ASR_en_4.2.0_3.0_1664024690847.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wolof_ASR', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wolof_ASR", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wolof_ASR| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: General model for table detection author: John Snow Labs name: general_model_table_detection_v2 date: 2023-01-10 tags: [en, licensed] task: OCR Table Detection language: en nav_key: models edition: Visual NLP 4.1.0 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description For table detection it is proposed the use of CascadeTabNet. It is a Cascade mask Region-based CNN High-Resolution Network (Cascade mask R-CNN HRNet) based model that detects tables on input images. The model is evaluated on ICDAR 2013, ICDAR 2019 and TableBank public datasets. It achieved 3rd rank in ICDAR 2019 post-competition results for table detection while attaining the best accuracy results for the ICDAR 2013 and TableBank dataset. Here it is used the CascadeTabNet general model for table detection inspired by https://arxiv.org/abs/2004.12629 ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/jupyter/Cards/SparkOcrImageTableDetection.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/general_model_table_detection_v2_en_3.3.0_3.0_1623301511401.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/general_model_table_detection_v2_en_3.3.0_3.0_1623301511401.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) table_detector = ImageTableDetector.pretrained("general_model_table_detection_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("table_regions") draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("table_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.red) pipeline = PipelineModel(stages=[ binary_to_image, table_detector, draw_regions ]) # Download image: # !wget -q https://github.com/JohnSnowLabs/spark-ocr-workshop/raw/4.0.0-release-candidate/jupyter/data/tab_images/cTDaR_t10168.jpg imagePath = "cTDaR_t10168.jpg" image_df = spark.read.format("binaryFile").load(imagePath) result = pipeline.transform(image_df) ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val table_detector = ImageTableDetector.pretrained("general_model_table_detection_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("table_regions") val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("table_regions") .setOutputCol("image_with_regions") .setRectColor(Color.red) val pipeline = new PipelineModel().setStages(Array( binary_to_image, table_detector, draw_regions)) # Download image: # !wget -q https://github.com/JohnSnowLabs/spark-ocr-workshop/raw/4.0.0-release-candidate/jupyter/data/tab_images/cTDaR_t10168.jpg val imagePath = "cTDaR_t10168.jpg" val image_df = spark.read.format("binaryFile").load(imagePath) val result = pipeline.transform(image_df) ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image5.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image5_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ocr_table_detection_general_model| |Type:|ocr| |Compatibility:|Visual NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Output Labels:|[table regions]| |Language:|en| ## Benchmarking ```bash 3rd rank in ICDAR 2019 post-competition 1st rank in ICDAR 2013 and TableBank dataset ``` --- layout: model title: Relation Extraction between different oncological entity types (granular version) author: John Snow Labs name: re_oncology_granular_wip date: 2022-09-27 tags: [licensed, clinical, oncology, en, relation_extraction, temporal, test, biomarker, anatomy] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Using this relation extraction model, four relation types can be identified: is_date_of (between date entities and other clinical entities), is_size_of (between Tumor_Finding and Tumor_Size), is_location_of (between anatomical entities and other entities) and is_finding_of (between test entities and their results). ## Predicted Entities `is_size_of`, `is_finding_of`, `is_date_of`, `is_location_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_oncology_granular_wip_en_4.0.0_3.0_1664301874672.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_oncology_granular_wip_en_4.0.0_3.0_1664301874672.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use realation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash chunk1 entity1 chunk2 entity2 relation confidence mastectomy Cancer_Surgery two months ago Relative_Date is_date_of 0.91336143 3 cm Tumor_Size mass Tumor_Finding is_size_of 0.96745735 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_oncology_granular_wip| |Type:|re| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Size:|267.2 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash relation recall precision f1 is_size_of 0.96 0.73 0.83 O 0.67 0.93 0.78 is_finding_of 0.94 0.75 0.83 is_date_of 0.94 0.54 0.69 is_location_of 0.94 0.81 0.87 macro-avg 0.89 0.75 0.80 ``` --- layout: model title: Legal Social Affairs Document Classifier (EURLEX) author: John Snow Labs name: legclf_social_affairs_bert date: 2023-03-06 tags: [en, legal, classification, clauses, social_affairs, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_social_affairs_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Social_Affairs or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Social_Affairs`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_social_affairs_bert_en_1.0.0_3.0_1678111486819.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_social_affairs_bert_en_1.0.0_3.0_1678111486819.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_social_affairs_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Social_Affairs]| |[Other]| |[Other]| |[Social_Affairs]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_social_affairs_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.84 0.81 0.83 59 Social_Affairs 0.81 0.84 0.82 56 accuracy - - 0.83 115 macro-avg 0.83 0.83 0.83 115 weighted-avg 0.83 0.83 0.83 115 ``` --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-32-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6_en_4.0.0_3.0_1655732701264.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6_en_4.0.0_3.0_1655732701264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_32d_seed_6").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|417.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-32-finetuned-squad-seed-6 --- layout: model title: ELECTRA Embeddings(ELECTRA Small) author: John Snow Labs name: electra_large_uncased date: 2020-08-27 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ELECTRA is a BERT-like model that is pre-trained as a discriminator in a set-up resembling a generative adversarial network (GAN). It was originally published by: Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning: [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/forum?id=r1xMH1BtvB), ICLR 2020. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_large_uncased_en_2.6.0_2.4_1598485645331.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_large_uncased_en_2.6.0_2.4_1598485645331.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("electra_large_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("electra_large_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.electra.large_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_electra_large_uncased_embeddings token [0.1289837807416916, -0.18811583518981934, 0.0... I [-0.02723774127662182, 0.0757141262292862, 0.3... love [0.4146347939968109, -0.31447598338127136, -0.... NLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_large_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/google/electra_large/2 --- layout: model title: BERT Sentence Embeddings trained on Wikipedia and BooksCorpus and fine-tuned on MNLI author: John Snow Labs name: sent_bert_wiki_books_mnli date: 2021-08-31 tags: [en, open_source, sentence_embeddings, wikipedia_dataset, books_corpus_dataset, mnli_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/wiki_books/1 and fine-tuned on MNLI. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. This model is intended to be used for a variety of English NLP tasks. The pre-training data contains more formal text and the model may not generalize to more colloquial text such as social media or messages. This model is fine-tuned on the MNLI and is recommended for use in natural language inference tasks. The MNLI fine-tuning task is a textual entailment task and includes data from a range of genres of spoken and written text. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_mnli_en_3.2.0_3.0_1630412088632.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_mnli_en_3.2.0_3.0_1630412088632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_mnli", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_mnli", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.wiki_books_mnli').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_wiki_books_mnli| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [Wikipedia dataset](https://dumps.wikimedia.org/) [2]: [BooksCorpus dataset](http://yknzhu.wixsite.com/mbweb) [3]: [MNLI dataset](https://cims.nyu.edu/~sbowman/multinli/) This Model has been imported from: https://tfhub.dev/google/experts/bert/wiki_books/mnli/2 --- layout: model title: Sentence Entity Resolver for SNOMED codes (procedures and measurements) author: John Snow Labs name: sbiobertresolve_snomed_procedures_measurements date: 2021-11-11 tags: [en, entity_resolution, licensed, clinical] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps medical entities to SNOMED codes using Sentence Bert Embeddings. The corpus of this model includes `Procedures` and `Measurement` domains. ## Predicted Entities `SNOMED Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_procedures_measurements_en_3.3.0_3.0_1636595616820.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_procedures_measurements_en_3.3.0_3.0_1636595616820.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli','en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_snomed_procedures_measurements", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("snomed_code") pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, resolver]) l_model = LightPipeline(pipelineModel) result = l_model.fullAnnotate(['coronary calcium score', 'heart surgery', 'ct scan', 'bp value']) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_procedures_measurements", "en", "clinical/models) .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("snomed_code") val pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, resolver)) val l_model = LightPipeline(pipelineModel) val result = l_model.fullAnnotate(Array("coronary calcium score", "heart surgery", "ct scan", "bp value")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed.procedures_measurements").predict("""coronary calcium score""") ```

## Results ```bash | | chunk | code | code_description | all_k_code_desc | all_k_codes | |---:|:-----------------------|----------:|:------------------------------|:--------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | coronary calcium score | 450360000 | Coronary artery calcium score | ['450360000', '450734004', '1086491000000104', '1086481000000101', '762241007'] | ['Coronary artery calcium score', 'Coronary artery calcium score', 'Dundee Coronary Risk Disk score', 'Dundee Coronary Risk rank', 'Dundee Coronary Risk Disk'] | | 1 | heart surgery | 2598006 | Open heart surgery | ['2598006', '64915003', '119766003', '34068001', '233004008'] | ['Open heart surgery', 'Operation on heart', 'Heart reconstruction', 'Heart valve replacement', 'Coronary sinus operation'] | | 2 | ct scan | 303653007 | CT of head | ['303653007', '431864000', '363023007', '418272005', '241577003'] | ['CT of head', 'CT guided injection', 'CT of site', 'CT angiography', 'CT of spine'] | | 3 | bp value | 75367002 | Blood pressure | ['75367002', '6797001', '723232008', '46973005', '427732000'] | ['Blood pressure', 'Mean blood pressure', 'Average blood pressure', 'Blood pressure taking', 'Speed of blood pressure response'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_procedures_measurements| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_chunk_embeddings]| |Output Labels:|[output]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on `SNOMED` code dataset with `sbiobert_base_cased_mli` sentence embeddings. --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from aytugkaya) author: John Snow Labs name: xlmroberta_ner_aytugkaya_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `aytugkaya`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_aytugkaya_base_finetuned_panx_de_4.1.0_3.0_1660431282851.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_aytugkaya_base_finetuned_panx_de_4.1.0_3.0_1660431282851.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_aytugkaya_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_aytugkaya_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_aytugkaya_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/aytugkaya/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Korean BertForQuestionAnswering model (from ainize) author: John Snow Labs name: bert_qa_ainize_klue_bert_base_mrc date: 2022-06-02 tags: [ko, open_source, question_answering, bert] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `klue-bert-base-mrc` is a Korean model orginally trained by `ainize`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_ainize_klue_bert_base_mrc_ko_4.0.0_3.0_1654188058903.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_ainize_klue_bert_base_mrc_ko_4.0.0_3.0_1654188058903.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_ainize_klue_bert_base_mrc","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_ainize_klue_bert_base_mrc","ko") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.klue.bert.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_ainize_klue_bert_base_mrc| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ko| |Size:|413.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ainize/klue-bert-base-mrc - https://ainize.ai/ - https://main-klue-mrc-bert-scy6500.endpoint.ainize.ai/ - https://ainize.ai/scy6500/KLUE-MRC-BERT?branch=main - https://ainize.ai/teachable-nlp - https://link.ainize.ai/3FjvBVn --- layout: model title: Part of Speech for Bihari author: John Snow Labs name: pos_ud_bhtb date: 2021-03-09 tags: [part_of_speech, open_source, bihari, pos_ud_bhtb, bh] task: Part of Speech Tagging language: bh edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - NOUN - CCONJ - ADJ - PUNCT - ADP - VERB - PROPN - NUM - AUX - DET - PRON - SCONJ - PART - ADV - INTJ - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bhtb_bh_3.0.0_3.0_1615292414604.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bhtb_bh_3.0.0_3.0_1615292414604.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_bhtb", "bh") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Hello from John Snow Labs!']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_bhtb", "bh") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hello from John Snow Labs!").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs!""] token_df = nlu.load('bh.pos').predict(text) token_df ```

## Results ```bash token pos 0 Hello NOUN 1 from NOUN 2 John NOUN 3 Snow NOUN 4 Labs NOUN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bhtb| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|bh| --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_nl8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-nl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl8_en_4.3.0_3.0_1675118058956.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl8_en_4.3.0_3.0_1675118058956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_nl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_nl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_nl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|576.4 MB| ## References - https://huggingface.co/google/t5-efficient-large-nl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Hindi Part of Speech Tagger (from sagorsarker) author: John Snow Labs name: bert_pos_codeswitch_hineng_pos_lince date: 2022-05-09 tags: [bert, pos, part_of_speech, hi, open_source] task: Part of Speech Tagging language: hi edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `codeswitch-hineng-pos-lince` is a Hindi model orginally trained by `sagorsarker`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_codeswitch_hineng_pos_lince_hi_3.4.2_3.0_1652091720542.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_codeswitch_hineng_pos_lince_hi_3.4.2_3.0_1652091720542.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_codeswitch_hineng_pos_lince| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|hi| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sagorsarker/codeswitch-hineng-pos-lince - https://ritual.uh.edu/lince/home - https://github.com/sagorbrur/codeswitch --- layout: model title: English image_classifier_vit_rust_image_classification_4 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_4 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_4` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_4_en_4.1.0_3.0_1660167562540.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_4_en_4.1.0_3.0_1660167562540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_4", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_4", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_4| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from batterydata) author: John Snow Labs name: bert_qa_batterybert_uncased_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `batterybert-uncased-squad-v1` is a English model orginally trained by `batterydata`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_batterybert_uncased_squad_v1_en_4.0.0_3.0_1654179312654.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_batterybert_uncased_squad_v1_en_4.0.0_3.0_1654179312654.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_batterybert_uncased_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_batterybert_uncased_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_battery.bert.uncased.by_batterydata").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_batterybert_uncased_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/batterydata/batterybert-uncased-squad-v1 - https://github.com/ShuHuang/batterybert --- layout: model title: Extract temporal relations among clinical events (ReDL) author: John Snow Labs name: redl_temporal_events_biobert date: 2021-07-24 tags: [relation_extraction, en, clinical, licensed] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 2.4 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations between clinical events in terms of time. If an event occurred before, after, or overlaps another event. ## Predicted Entities `AFTER`, `BEFORE`, `OVERLAP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CLINICAL_EVENTS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_temporal_events_biobert_en_3.0.3_2.4_1627121501681.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_temporal_events_biobert_en_3.0.3_2.4_1627121501681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------+-------------+-------------+-----------+-----------+-------------+-------------+-----------+------------+----------+ |relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +--------+-------------+-------------+-----------+-----------+-------------+-------------+-----------+------------+----------+ | BEFORE| OCCURRENCE| 7| 15| diagnosed| PROBLEM| 22| 27| cancer|0.78168863| | OVERLAP| PROBLEM| 22| 27| cancer| DATE| 32| 35| 1991| 0.8492274| | AFTER| OCCURRENCE| 51| 58| admitted|CLINICAL_DEPT| 63| 73| Mayo Clinic|0.85629463| | BEFORE| OCCURRENCE| 51| 58| admitted| OCCURRENCE| 91| 100| discharged| 0.6843513| | OVERLAP|CLINICAL_DEPT| 63| 73|Mayo Clinic| DATE| 78| 85| May 2000| 0.7844673| | BEFORE|CLINICAL_DEPT| 63| 73|Mayo Clinic| OCCURRENCE| 91| 100| discharged|0.60411876| | OVERLAP|CLINICAL_DEPT| 63| 73|Mayo Clinic| DATE| 105| 116|October 2001| 0.540761| | BEFORE| DATE| 78| 85| May 2000| OCCURRENCE| 91| 100| discharged| 0.6042761| | OVERLAP| DATE| 78| 85| May 2000| DATE| 105| 116|October 2001|0.64867175| | BEFORE| OCCURRENCE| 91| 100| discharged| DATE| 105| 116|October 2001| 0.5302478| +--------+-------------+-------------+-----------+-----------+-------------+-------------+-----------+------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_temporal_events_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on temporal clinical events benchmark dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support AFTER 0.332 0.655 0.440 2123 BEFORE 0.868 0.908 0.887 13817 OVERLAP 0.887 0.733 0.802 7860 Avg. 0.695 0.765 0.710 - ``` --- layout: model title: Chinese Bert Embeddings (6-layer) author: John Snow Labs name: bert_embeddings_rbt6 date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `rbt6` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt6_zh_3.4.2_3.0_1649669338288.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt6_zh_3.4.2_3.0_1649669338288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt6| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|224.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt6 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Part of Speech for Czech author: John Snow Labs name: pos_cac date: 2021-03-23 tags: [cs, open_source, pos] supported: true task: Part of Speech Tagging language: cs edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - INTJ - NOUN - NUM - PART - PRON - PROPN - PUNCT - SCONJ - SYM - VERB {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_cac_cs_2.7.5_2.4_1616507827439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_cac_cs_2.7.5_2.4_1616507827439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_cac", "cs")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([["'Soustředit všechny propagační a agitační prostředky na vytvoření veřejného mínění , které bude podporovat přesnou , tvořivou a iniciativní práci ."]], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_cac", "cs") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer , pos)) val data = Seq("'Soustředit všechny propagační a agitační prostředky na vytvoření veřejného mínění , které bude podporovat přesnou , tvořivou a iniciativní práci .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Soustředit všechny propagační a agitační prostředky na vytvoření veřejného mínění , které bude podporovat přesnou , tvořivou a iniciativní práci .""] token_df = nlu.load('cs.pos.cac').predict(text) token_df ```

## Results ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ |text |result | +---------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ |'Soustředit všechny propagační a agitační prostředky na vytvoření veřejného mínění , které bude podporovat přesnou , tvořivou a iniciativní práci .|[NOUN, VERB, DET, ADJ, CCONJ, ADJ, NOUN, ADP, NOUN, ADJ, NOUN, PUNCT, DET, AUX, VERB, ADJ, PUNCT, ADJ, CCONJ, ADJ, NOUN, PUNCT]| +---------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_cac| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|cs| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.96 | 0.97 | 0.97 | 1597 | | ADP | 1.00 | 0.99 | 1.00 | 1031 | | ADV | 0.99 | 0.97 | 0.98 | 552 | | AUX | 0.93 | 0.97 | 0.95 | 302 | | CCONJ | 1.00 | 1.00 | 1.00 | 575 | | DET | 1.00 | 0.98 | 0.99 | 418 | | INTJ | 1.00 | 1.00 | 1.00 | 1 | | NOUN | 0.97 | 0.98 | 0.97 | 3023 | | NUM | 1.00 | 0.97 | 0.99 | 103 | | PART | 0.99 | 0.92 | 0.96 | 92 | | PRON | 0.98 | 0.97 | 0.98 | 353 | | PROPN | 0.93 | 0.91 | 0.92 | 138 | | PUNCT | 1.00 | 1.00 | 1.00 | 1423 | | SCONJ | 0.99 | 0.99 | 0.99 | 175 | | SYM | 1.00 | 1.00 | 1.00 | 39 | | VERB | 0.97 | 0.94 | 0.95 | 1040 | | accuracy | | | 0.98 | 10862 | | macro avg | 0.98 | 0.97 | 0.98 | 10862 | | weighted avg | 0.98 | 0.98 | 0.98 | 10862 | ``` --- layout: model title: English BertForQuestionAnswering model (from andresestevez) author: John Snow Labs name: bert_qa_andresestevez_bert_finetuned_squad_accelerate date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model orginally trained by `andresestevez`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_andresestevez_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535829208.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_andresestevez_bert_finetuned_squad_accelerate_en_4.0.0_3.0_1654535829208.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_andresestevez_bert_finetuned_squad_accelerate","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_andresestevez_bert_finetuned_squad_accelerate","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_andresestevez").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_andresestevez_bert_finetuned_squad_accelerate| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/andresestevez/bert-finetuned-squad-accelerate --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Portuguese (WikiNER 6B 300) author: John Snow Labs name: wikiner_6B_300 date: 2020-05-10 task: Named Entity Recognition language: pt edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [ner, pt, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 6B 300 is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_PT){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_PT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_pt_2.5.0_2.4_1588495233641.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_pt_2.5.0_2.4_1588495233641.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PER | |Ele |MISC | |Microsoft Corporation |ORG | |Durante |MISC | |Microsoft |ORG | |Gates |PER | |CEO |ORG | |Ele |MISC | |Nascido |MISC | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |Novo México |LOC | |Gates |PER | |CEO |ORG | |Gates |PER | |Gates |PER | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Case sensitive:|false| {:.h2_title} ## Data Source The model was trained based on data from [https://pt.wikipedia.org](https://pt.wikipedia.org) --- layout: model title: Stopwords Remover for Catalan language (280 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ca, open_source] task: Stop Words Removal language: ca edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ca_3.4.1_3.0_1646673133236.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ca_3.4.1_3.0_1646673133236.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------+ |result | +--------+ |[millor]| +--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ca| |Size:|2.1 KB| --- layout: model title: Pipeline to Detect Drug Chemicals author: John Snow Labs name: ner_drugs_large_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_drugs_large](https://nlp.johnsnowlabs.com/2021/03/31/ner_drugs_large_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugs_large_pipeline_en_4.3.0_3.2_1678867384370.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugs_large_pipeline_en_4.3.0_3.2_1678867384370.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_drugs_large_pipeline", "en", "clinical/models") text = '''The patient is a 40-year-old white male who presents with a chief complaint of 'chest pain'. The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. He has been advised Aspirin 81 milligrams QDay. Humulin N. insulin 50 units in a.m. HCTZ 50 mg QDay. Nitroglycerin 1/150 sublingually PRN chest pain..''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_drugs_large_pipeline", "en", "clinical/models") val text = "The patient is a 40-year-old white male who presents with a chief complaint of 'chest pain'. The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. He has been advised Aspirin 81 milligrams QDay. Humulin N. insulin 50 units in a.m. HCTZ 50 mg QDay. Nitroglycerin 1/150 sublingually PRN chest pain.." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.drugs_large.pipeline").predict("""The patient is a 40-year-old white male who presents with a chief complaint of 'chest pain'. The patient is diabetic and has a prior history of coronary artery disease. The patient presents today stating that his chest pain started yesterday evening and has been somewhat intermittent. He has been advised Aspirin 81 milligrams QDay. Humulin N. insulin 50 units in a.m. HCTZ 50 mg QDay. Nitroglycerin 1/150 sublingually PRN chest pain..""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:---------------------------------|--------:|------:|:------------|-------------:| | 0 | Aspirin 81 milligrams | 306 | 326 | DRUG | 0.8401 | | 1 | Humulin N | 334 | 342 | DRUG | 0.95755 | | 2 | insulin 50 units | 345 | 360 | DRUG | 0.847067 | | 3 | HCTZ 50 mg | 370 | 379 | DRUG | 0.875567 | | 4 | Nitroglycerin 1/150 sublingually | 387 | 418 | DRUG | 0.845967 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Basque RoBERTa Embeddings Cased model (from mrm8488) author: John Snow Labs name: roberta_embeddings_robasqu date: 2022-07-14 tags: [eu, open_source, roberta, embeddings] task: Embeddings language: eu edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RoBasquERTa` is a Basque model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robasqu_eu_4.0.0_3.0_1657809995026.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_robasqu_eu_4.0.0_3.0_1657809995026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = RoBertaEmbeddings.pretrained("roberta_embeddings_robasqu","eu") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = RoBertaEmbeddings.pretrained("roberta_embeddings_robasqu","eu") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_robasqu| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|eu| |Size:|313.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/mrm8488/RoBasquERTa --- layout: model title: Indonesian BertForMaskedLM Base Cased model (from cahya) author: John Snow Labs name: bert_embeddings_base_indonesian_522m date: 2022-12-02 tags: [id, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: id edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-indonesian-522M` is a Indonesian model originally trained by `cahya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_indonesian_522m_id_4.2.4_3.0_1670017835135.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_indonesian_522m_id_4.2.4_3.0_1670017835135.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_indonesian_522m","id") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_indonesian_522m","id") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_indonesian_522m| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|id| |Size:|415.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/cahya/bert-base-indonesian-522M - https://github.com/cahya-wirawan/indonesian-language-models/tree/master/Transformers --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_4 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-4` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_4_en_4.3.0_3.0_1672767525583.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_4_en_4.3.0_3.0_1672767525583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-4 --- layout: model title: English asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves TFWav2Vec2ForCTC from tonyalves author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves` is a English model originally trained by tonyalves. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_en_4.2.0_3.0_1664108795578.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves_en_4.2.0_3.0_1664108795578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_colab_by_tonyalves| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Pipeline to Detect Mentions of General Medical Terms author: John Snow Labs name: ner_medmentions_coarse_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_medmentions_coarse](https://nlp.johnsnowlabs.com/2021/04/01/ner_medmentions_coarse_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_medmentions_coarse_pipeline_en_3.4.1_3.0_1647870771676.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_medmentions_coarse_pipeline_en_3.4.1_3.0_1647870771676.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_medmentions_coarse_pipeline", "en", "clinical/models") pipeline.annotate("EXAMPLE MEDICAL TEXT") ``` ```scala val pipeline = new PretrainedPipeline("ner_medmentions_coarse_pipeline", "en", "clinical/models") pipeline.annotate("EXAMPLE MEDICAL TEXT") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.medmentions_coarse.pipeline").predict("""EXAMPLE MEDICAL TEXT""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_medmentions_coarse_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265910` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910_en_4.0.0_3.0_1655985771660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910_en_4.0.0_3.0_1655985771660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265910").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265910| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265910 --- layout: model title: Multilingual T5ForConditionalGeneration Cased model (from csebuetnlp) author: John Snow Labs name: t5_banglat5_nmt_en2bn date: 2023-01-30 tags: [bn, en, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `banglat5_nmt_en_bn` is a Multilingual model originally trained by `csebuetnlp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_banglat5_nmt_en2bn_xx_4.3.0_3.0_1675100250892.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_banglat5_nmt_en2bn_xx_4.3.0_3.0_1675100250892.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_banglat5_nmt_en2bn","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_banglat5_nmt_en2bn","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_banglat5_nmt_en2bn| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|1.0 GB| ## References - https://huggingface.co/csebuetnlp/banglat5_nmt_en_bn - https://github.com/csebuetnlp/normalizer --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squad1.1-block-sparse-0.20-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1_en_4.0.0_3.0_1654181440387.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1_en_4.0.0_3.0_1654181440387.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_x1.16_f88.1_d8_unstruct_v1.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squad1.1_block_sparse_0.20_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|179.0 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squad1.1-block-sparse-0.20-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf - https://www.aclweb.org/anthology/N19-1423/ - https://arxiv.org/abs/2005.07683 --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_cline_techqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cline-techqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_techqa_en_4.0.0_3.0_1655727943280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_techqa_en_4.0.0_3.0_1655727943280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_cline_techqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_cline_techqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.techqa_cline.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cline_techqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/cline-techqa --- layout: model title: ICD10CM ChunkResolver author: John Snow Labs name: chunkresolve_icd10cm_diseases_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-28 task: Entity Resolution edition: Healthcare NLP 2.4.5 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities ICD10-CM Codes and their normalized definition with ``clinical_embeddings``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_diseases_clinical_en_2.4.5_2.4_1588105984876.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_diseases_clinical_en_2.4.5_2.4_1588105984876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... icd10cmResolver = ChunkEntityResolverModel.pretrained('chunkresolve_icd10cm_diseases_clinical', 'en', "clinical/models")\ .setEnableLevenshtein(True)\ .setNeighbours(200).setAlternatives(5).setDistanceWeights([3,3,2,0,0,7])\ .setInputCols('token', 'chunk_embs_jsl')\ .setOutputCol('icd10cm_resolution') pipeline_icd10 = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, icd10cmResolver]) empty_df = spark.createDataFrame([[""]]).toDF("text") data = ["""This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret's Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU"""] pipeline_model = pipeline_icd10.fit(empty_df) light_pipeline = LightPipeline(pipeline_model) result = light_pipeline.annotate(data) ``` ```scala ... val icd10cmResolver = ChunkEntityResolverModel.pretrained('chunkresolve_icd10cm_diseases_clinical', 'en', "clinical/models") .setEnableLevenshtein(True) .setNeighbours(200).setAlternatives(5).setDistanceWeights(Array(3,3,2,0,0,7)) .setInputCols('token', 'chunk_embs_jsl') .setOutputCol('icd10cm_resolution') val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, icd10cmResolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret's Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash | | coords | chunk | entity | icd10cm_opts | |---|-------------|-----------------------------|-----------|-------------------------------------------------------------------------------------------| | 0 | 2::499::506 | insomnia | Diagnosis | [(G4700, Insomnia, unspecified), (G4709, Other insomnia), (F5102, Adjustment insomnia)...]| | 1 | 4::83::109 | chronic renal insufficiency | Diagnosis | [(N185, Chronic kidney disease, stage 5), (N181, Chronic kidney disease, stage 1), (N1...]| | 2 | 4::120::128 | gastritis | Diagnosis | [(K2970, Gastritis, unspecified, without bleeding), (B9681, Helicobacter pylori [H. py...]| ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------------------| | Name: | chunkresolve_icd10cm_diseases_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD10CM Dataset Range: A000-N989 Except Neoplasms and Musculoskeletal https://www.icd10data.com/ICD10CM/Codes/ --- layout: model title: English Named Entity Recognition (from browndw) author: John Snow Labs name: bert_ner_docusco_bert date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `docusco-bert` is a English model orginally trained by `browndw`. ## Predicted Entities `Interactive`, `AcademicTerms`, `InformationChange`, `MetadiscourseCohesive`, `FirstPerson`, `InformationPlace`, `Updates`, `InformationChangeneritive`, `Reasoning`, `PublicTerms`, `Citation`, `Future`, `CitationHedged`, `InformationExnerition`, `Contingent`, `Strategic`, `PAD`, `CitationAuthority`, `Facilitate`, `Positive`, `ConfidenceHigh`, `InformationStates`, `AcademicWritingMoves`, `Uncertainty`, `SyntacticComplexity`, `Responsibility`, `Character`, `Narrative`, `MetadiscourseInteractive`, `InformationTopics`, `ConfidenceLow`, `ConfidenceHedged`, `ForceStressed`, `Negative`, `InformationChangeNegative`, `Description`, `Inquiry`, `InformationReportVerbs` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_docusco_bert_en_3.4.2_3.0_1652097061800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_docusco_bert_en_3.4.2_3.0_1652097061800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_docusco_bert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/browndw/docusco-bert - https://www.english-corpora.org/coca/ - https://www.cmu.edu/dietrich/english/research-and-publications/docuscope.html - https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=docuscope&btnG= - https://graphics.cs.wisc.edu/WP/vep/2017/02/14/guest-post-data-mining-king-lear/ - https://journals.sagepub.com/doi/full/10.1177/2055207619844865 - https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging) - https://www.english-corpora.org/coca/ - https://arxiv.org/pdf/1810.04805 --- layout: model title: Japanese ALBERT Embeddings (from ken11) author: John Snow Labs name: albert_embeddings_albert_base_japanese_v1 date: 2022-04-14 tags: [albert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-base-japanese-v1` is a Japanese model orginally trained by `ken11`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_japanese_v1_ja_3.4.2_3.0_1649954238475.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_japanese_v1_ja_3.4.2_3.0_1649954238475.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_base_japanese_v1| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|45.6 MB| |Case sensitive:|false| ## References - https://huggingface.co/ken11/albert-base-japanese-v1 - https://ken11.jp/blog/sentencepiece-tokenizer-bug - https://ja.wikipedia.org/wiki/Wikipedia:%E3%83%87%E3%83%BC%E3%82%BF%E3%83%99%E3%83%BC%E3%82%B9%E3%83%80%E3%82%A6%E3%83%B3%E3%83%AD%E3%83%BC%E3%83%89 - https://www.rondhuit.com/download.html#ldcc - https://github.com/google/sentencepiece - https://opensource.org/licenses/MIT --- layout: model title: Google's Tapas Table Understanding (Base, SQA) author: John Snow Labs name: table_qa_tapas_base_finetuned_sqa date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Base Has aggregation operations?: False ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_base_finetuned_sqa_en_4.2.0_3.0_1664530865349.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_base_finetuned_sqa_en_4.2.0_3.0_1664530865349.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_base_finetuned_sqa","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_32_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-32-finetuned-squad-seed-2 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becasv2_4 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becasv2-4` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_4_en_4.3.0_3.0_1672767755946.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becasv2_4_en_4.3.0_3.0_1672767755946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becasv2_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becasv2_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becasv2-4 --- layout: model title: Legal Taxes Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_taxes_bert date: 2023-03-05 tags: [en, legal, classification, clauses, taxes, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Taxes` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Taxes`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_taxes_bert_en_1.0.0_3.0_1678050662883.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_taxes_bert_en_1.0.0_3.0_1678050662883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_taxes_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Taxes]| |[Other]| |[Other]| |[Taxes]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_taxes_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.96 0.98 0.97 167 Taxes 0.97 0.96 0.96 134 accuracy - - 0.97 301 macro-avg 0.97 0.97 0.97 301 weighted-avg 0.97 0.97 0.97 301 ``` --- layout: model title: Igbo Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_igbo_finetuned_ner_igbo date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, ig, open_source] task: Named Entity Recognition language: ig edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-igbo-finetuned-ner-igbo` is a Igbo model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_igbo_finetuned_ner_igbo_ig_3.4.2_3.0_1652809440980.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_igbo_finetuned_ner_igbo_ig_3.4.2_3.0_1652809440980.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_igbo_finetuned_ner_igbo| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ig| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-igbo-finetuned-ner-igbo - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman/NER --- layout: model title: English asr_wav2vec2_base_100h_13K_steps TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: pipeline_asr_wav2vec2_base_100h_13K_steps date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_13K_steps` is a English model originally trained by patrickvonplaten. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_100h_13K_steps_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_13K_steps_en_4.2.0_3.0_1664101244680.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_100h_13K_steps_en_4.2.0_3.0_1664101244680.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_100h_13K_steps', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_100h_13K_steps", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_100h_13K_steps| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.2 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline to Detect clinical concepts (jsl_ner_wip_modifier_clinical) author: John Snow Labs name: jsl_ner_wip_modifier_clinical_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [jsl_ner_wip_modifier_clinical](https://nlp.johnsnowlabs.com/2021/04/01/jsl_ner_wip_modifier_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_modifier_clinical_pipeline_en_4.3.0_3.2_1678782821242.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_modifier_clinical_pipeline_en_4.3.0_3.2_1678782821242.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("jsl_ner_wip_modifier_clinical_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature..''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("jsl_ner_wip_modifier_clinical_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.wip_modifier_clinical.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature..""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.9817 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.9998 | | 2 | male | 38 | 41 | Gender | 0.9922 | | 3 | for 2 days | 48 | 57 | Duration | 0.6968 | | 4 | congestion | 62 | 71 | Symptom | 0.875 | | 5 | mom | 75 | 77 | Gender | 0.8156 | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | 0.2697 | | 7 | nares | 135 | 139 | External_body_part_or_region | 0.6216 | | 8 | she | 147 | 149 | Gender | 0.9965 | | 9 | mild problems with his breathing while feeding | 168 | 213 | Symptom | 0.444029 | | 10 | perioral cyanosis | 237 | 253 | Symptom | 0.3283 | | 11 | retractions | 258 | 268 | Symptom | 0.957 | | 12 | One day ago | 272 | 282 | RelativeDate | 0.646267 | | 13 | mom | 285 | 287 | Gender | 0.692 | | 14 | tactile temperature | 304 | 322 | Symptom | 0.20765 | | 15 | Tylenol | 345 | 351 | Drug | 0.9951 | | 16 | Baby | 354 | 357 | Age | 0.981 | | 17 | decreased p.o. intake | 377 | 397 | Symptom | 0.437375 | | 18 | His | 400 | 402 | Gender | 0.999 | | 19 | 20 minutes | 439 | 448 | Duration | 0.20415 | | 20 | q.2h. | 450 | 454 | Frequency | 0.6406 | | 21 | to 5 to 10 minutes | 456 | 473 | Duration | 0.12444 | | 22 | his | 488 | 490 | Gender | 0.9904 | | 23 | respiratory congestion | 492 | 513 | Symptom | 0.5294 | | 24 | He | 516 | 517 | Gender | 0.9989 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_modifier_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from Papiamento to English author: John Snow Labs name: opus_mt_pap_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pap, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `pap` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_pap_en_xx_2.7.0_2.4_1609167979033.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_pap_en_xx_2.7.0_2.4_1609167979033.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_pap_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_kv128 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-kv128` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_kv128_en_4.3.0_3.0_1675121257275.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_kv128_en_4.3.0_3.0_1675121257275.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_kv128","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_kv128","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_kv128| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|183.9 MB| ## References - https://huggingface.co/google/t5-efficient-small-kv128 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Fast Neural Machine Translation Model from Slavic Languages to English author: John Snow Labs name: opus_mt_sla_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, sla, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `sla` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_sla_en_xx_2.7.0_2.4_1609164230956.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_sla_en_xx_2.7.0_2.4_1609164230956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_sla_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Understanding Restriction Level of Assignment Clauses(Bert) author: John Snow Labs name: legclf_nda_assigments_bert date: 2023-05-17 tags: [en, legal, licensed, bert, nda, classification, assigments, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Given a clause classified as `ASSIGNMENT ` using the `legmulticlf_mnda_sections_paragraph_other` classifier, you can subclassify the sentences as `PERMISSIVE_ASSIGNMENT`, `RESTRICTIVE_ASSIGNMENT` or `OTHER` from it using the `legclf_nda_assigments_bert` model. It has been trained with the SOTA approach. ## Predicted Entities `PERMISSIVE_ASSIGNMENT`, `RESTRICTIVE_ASSIGNMENT`, `OTHER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_nda_assigments_bert_en_1.0.0_3.0_1684350248553.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_nda_assigments_bert_en_1.0.0_3.0_1684350248553.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------------------------+----------------------+ | text| class| +--------------------------------------------------------------------------------+----------------------+ |This Agreement will be binding upon and inure to the benefit of each Party an...| PERMISSIVE_ASSIGNMENT| |All notices and other communications provided for in this Agreement and the o...| OTHER| |This Agreement is a personal contract for XCorp, and the rights and interests...|RESTRICTIVE_ASSIGNMENT| +--------------------------------------------------------------------------------+----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_nda_assigments_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## Sample text from the training dataset In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support OTHER 0.98 1.00 0.99 124 PERMISSIVE_ASSIGNMENT 1.00 0.93 0.97 15 RESTRICTIVE_ASSIGNMENT 1.00 0.96 0.98 25 accuracy - - 0.99 164 macro avg 0.99 0.96 0.98 164 weighted avg 0.99 0.99 0.99 164 ``` --- layout: model title: English asr_wav2vec2_base_checkpoint_14 TFWav2Vec2ForCTC from jiobiala24 author: John Snow Labs name: pipeline_asr_wav2vec2_base_checkpoint_14 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_checkpoint_14` is a English model originally trained by jiobiala24. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_checkpoint_14_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_checkpoint_14_en_4.2.0_3.0_1664114924936.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_checkpoint_14_en_4.2.0_3.0_1664114924936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_checkpoint_14', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_checkpoint_14", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_checkpoint_14| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.1 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Medical Spell Checker author: John Snow Labs name: spellcheck_clinical date: 2021-02-16 task: Spell Check language: en nav_key: models edition: Spark NLP 2.7.2 spark_version: 2.4 tags: [spelling, spellchecker, clinical, orthographic, spell_checker, medical_spell_checker, spelling_corrector, en, licensed] supported: true annotator: SpellCheckModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Contextual Spell Checker is a sequence-to-sequence model that detects and corrects spelling errors in your input text. It's based on Levenshtein Automaton for generating candidate corrections and a Neural Language Model for ranking corrections. This model has been trained in a dataset containing data from different sources; MTSamples, i2b2 clinical notes, and PubMed. You can download the model that comes fully pretrained and ready to use. However, you can still customize it further without the need for re-training a new model from scratch. This can be accomplished by providing custom definitions for the word classes the model has been trained on, namely Dates, Numbers, Ages, Units, and Medications. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_en_2.7.2_2.4_1613505168792.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/spellcheck_clinical_en_2.7.2_2.4_1613505168792.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In order to use this model, you need to setup a pipeline and feed tokens.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = RecursiveTokenizer()\ .setInputCols(["document"])\ .setOutputCol("token")\ .setPrefixes(["\"", "(", "[", "\n"])\ .setSuffixes([".", ",", "?", ")","!", "'s"]) spellModel = ContextSpellCheckerModel\ .pretrained('spellcheck_clinical', 'en', 'clinical/models')\ .setInputCols("token")\ .setOutputCol("checked") finisher = Finisher()\ .setInputCols("checked") pipeline = Pipeline( stages = [ documentAssembler, tokenizer, spellModel, finisher ]) empty_ds = spark.createDataFrame([[""]]).toDF("text") lp = LightPipeline(pipeline.fit(empty_ds)) example = ["Witth the hell of phisical terapy the patient was imbulated and on posoperative, the impatient tolerating a post curgical soft diet.", "With paint wel controlled on orall pain medications, she was discharged too reihabilitation facilitay.", "She is to also call the ofice if she has any ever greater than 101, or leeding form the surgical wounds.", "Abdomen is sort, nontender, and nonintended.", "Patient not showing pain or any wealth problems.", "No cute distress" ] lp.annotate(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.spell.clinical").predict(""") pipeline = Pipeline( stages = [ documentAssembler, tokenizer, spellModel, finisher ]) empty_ds = spark.createDataFrame([[""") ```

## Results ```bash [{'checked': ['With', 'the', 'help', 'of', 'physical', 'therapy', 'the', 'patient', 'was', 'ambulated', 'and', 'on', 'postoperative', ',', 'the', 'patient', 'tolerating', 'a', 'post', 'surgical', 'soft', 'diet', '.']}, {'checked': ['With', 'pain', 'well', 'controlled', 'on', 'oral', 'pain', 'medications', ',', 'she', 'was', 'discharged', 'to', 'rehabilitation', 'facility', '.']}, {'checked': ['She', 'is', 'to', 'also', 'call', 'the', 'office', 'if', 'she', 'has', 'any', 'fever', 'greater', 'than', '101', ',', 'or', 'bleeding', 'from', 'the', 'surgical', 'wounds', '.']}, {'checked': ['Abdomen', 'is', 'soft', ',', 'nontender', ',', 'and', 'nondistended', '.']}, {'checked': ['Patient', 'not', 'showing', 'pain', 'or', 'any', 'health', 'problems', '.']}, {'checked': ['No', 'acute', 'distress']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_clinical| |Compatibility:|Spark NLP 2.7.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token]| |Language:|en| ## Data Source MTSamples, i2b2 clinical notes, and PubMed. --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_company_vs_all_902129475 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-company_vs_all-902129475` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Company`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_vs_all_902129475_en_4.3.1_3.0_1678134115695.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_vs_all_902129475_en_4.3.1_3.0_1678134115695.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_vs_all_902129475","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_vs_all_902129475","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_company_vs_all_902129475| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-company_vs_all-902129475 --- layout: model title: English BertForQuestionAnswering Uncased model (from husnu) author: John Snow Labs name: bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xtremedistil-l6-h256-uncased-TQUAD-finetuned_lr-2e-05_epochs-6` is a English model originally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6_en_4.0.0_3.0_1657193243137.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6_en_4.0.0_3.0_1657193243137.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|47.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/husnu/xtremedistil-l6-h256-uncased-TQUAD-finetuned_lr-2e-05_epochs-6 --- layout: model title: Fast and Accurate Language Identification - 21 Languages (BIGRU) author: John Snow Labs name: ld_tatoeba_bigru_21 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using BiGRU architectures in TensorFlow/Keras. The model is trained on Tatoeba dataset with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias) This model can detect the following languages: `Bulgarian`, `Czech`, `Danish`, `German`, `Greek`, `English`, `Estonian`, `Finnish`, `French`, `Hungarian`, `Italian`, `Lithuanian`, `Latvian`, `Dutch`, `Polish`, `Portuguese`, `Romanian`, `Slovak`, `Slovenian`, `Spanish`, `Swedish`. ## Predicted Entities `bg`, `cs`, `da`, `de`, `el`, `en`, `et`, `fi`, `fr`, `hu`, `it`, `lt`, `lv`, `nl`, `pl`, `pt`, `ro`, `sk`, `sl`, `es`, `sv`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_tatoeba_bigru_21_xx_2.7.0_2.4_1607183021248.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_tatoeba_bigru_21_xx_2.7.0_2.4_1607183021248.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_tatoeba_bigru_21", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_tatoeba_bigru_21", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_21.bigru').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_tatoeba_bigru_21| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | el| 1000| 1000| 1.0| | pt| 1000| 1000| 1.0| | fr| 1000| 1000| 1.0| | it| 1000| 1000| 1.0| | de| 1000| 1000| 1.0| | es| 1000| 1000| 1.0| | nl| 1000| 999| 0.999| | en| 1000| 999| 0.999| | da| 1000| 999| 0.999| | fi| 1000| 998| 0.998| | pl| 914| 906|0.9912472647702407| | bg| 1000| 990| 0.99| | hu| 880| 870|0.9886363636363636| | ro| 784| 773| 0.985969387755102| | sv| 1000| 985| 0.985| | lt| 1000| 983| 0.983| | cs| 1000| 979| 0.979| | lv| 916| 896|0.9781659388646288| | sk| 1000| 969| 0.969| | et| 928| 888|0.9568965517241379| | sl| 914| 861|0.9420131291028446| +--------+-----+-------+------------------+ +-------+--------------------+ |summary| precision| +-------+--------------------+ | count| 21| | mean| 0.9878061255168247| | stddev|0.015811719210146295| | min| 0.9420131291028446| | max| 1.0| +-------+--------------------+ ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from vennify) author: John Snow Labs name: t5_base_grammar_correction date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-grammar-correction` is a English model originally trained by `vennify`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_grammar_correction_en_4.3.0_3.0_1675109588406.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_grammar_correction_en_4.3.0_3.0_1675109588406.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_grammar_correction","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_grammar_correction","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_grammar_correction| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|904.6 MB| ## References - https://huggingface.co/vennify/t5-base-grammar-correction - https://github.com/EricFillion/happy-transformer - https://arxiv.org/abs/1702.04066 - https://www.vennify.ai/fine-tune-grammar-correction/ --- layout: model title: Translate Bantu languages to English Pipeline author: John Snow Labs name: translate_bnt_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bnt, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bnt` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bnt_en_xx_2.7.0_2.4_1609698899237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bnt_en_xx_2.7.0_2.4_1609698899237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bnt_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bnt_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bnt.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bnt_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from English to Kongo author: John Snow Labs name: opus_mt_en_kg date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, kg, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `kg` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_kg_xx_2.7.0_2.4_1609167529740.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_kg_xx_2.7.0_2.4_1609167529740.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_kg| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191078345.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191078345.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_hier_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: English asr_wav2vec2_base_timit_moaiz_exp1 TFWav2Vec2ForCTC from moaiz237 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_moaiz_exp1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_moaiz_exp1` is a English model originally trained by moaiz237. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_moaiz_exp1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_moaiz_exp1_en_4.2.0_3.0_1664036388529.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_moaiz_exp1_en_4.2.0_3.0_1664036388529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_moaiz_exp1', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_moaiz_exp1", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_moaiz_exp1| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Recognize Entities DL Pipeline for Portuguese - Medium author: John Snow Labs name: entity_recognizer_md date: 2021-03-22 tags: [open_source, portuguese, entity_recognizer_md, pipeline, pt] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: pt edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_pt_3.0.0_3.0_1616449413384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_md_pt_3.0.0_3.0_1616449413384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_md', lang = 'pt') annotations = pipeline.fullAnnotate(""Olá de John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_md", lang = "pt") val result = pipeline.fullAnnotate("Olá de John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Olá de John Snow Labs! ""] result_df = nlu.load('pt.ner.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:----------------------------|:---------------------------|:---------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Olá de John Snow Labs! '] | ['Olá de John Snow Labs!'] | ['Olá', 'de', 'John', 'Snow', 'Labs!'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|pt| --- layout: model title: English asr_xlsr_punctuation TFWav2Vec2ForCTC from boris author: John Snow Labs name: asr_xlsr_punctuation date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_punctuation` is a English model originally trained by boris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xlsr_punctuation_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xlsr_punctuation_en_4.2.0_3.0_1664020729832.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xlsr_punctuation_en_4.2.0_3.0_1664020729832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xlsr_punctuation", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xlsr_punctuation", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xlsr_punctuation| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from bugdaryan) author: John Snow Labs name: distilbert_qa_bugdaryan_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `bugdaryan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_bugdaryan_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770282430.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_bugdaryan_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770282430.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bugdaryan_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_bugdaryan_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_bugdaryan_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/bugdaryan/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate Cebuano to English Pipeline author: John Snow Labs name: translate_ceb_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ceb, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ceb` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ceb_en_xx_2.7.0_2.4_1609691069990.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ceb_en_xx_2.7.0_2.4_1609691069990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ceb_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ceb_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ceb.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ceb_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Large Uncased model (from tiennvcs) author: John Snow Labs name: bert_qa_large_uncased_finetuned_vi_infovqa date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-finetuned-vi-infovqa` is a English model originally trained by `tiennvcs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_large_uncased_finetuned_vi_infovqa_en_4.0.0_3.0_1657187434802.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_large_uncased_finetuned_vi_infovqa_en_4.0.0_3.0_1657187434802.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_large_uncased_finetuned_vi_infovqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_large_uncased_finetuned_vi_infovqa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_large_uncased_finetuned_vi_infovqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tiennvcs/bert-large-uncased-finetuned-vi-infovqa --- layout: model title: English asr_wav2vec2_base_timit_demo_colab66 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab66 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab66` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab66_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab66_en_4.2.0_3.0_1664024863951.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab66_en_4.2.0_3.0_1664024863951.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab66', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab66", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab66| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Cased model (from AyushPJ) author: John Snow Labs name: distilbert_qa_test_squad_trained_finetuned date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `test-squad-trained-finetuned-squad` is a English model originally trained by `AyushPJ`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_test_squad_trained_finetuned_en_4.3.0_3.0_1672775755501.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_test_squad_trained_finetuned_en_4.3.0_3.0_1672775755501.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_test_squad_trained_finetuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_test_squad_trained_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_test_squad_trained_finetuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AyushPJ/test-squad-trained-finetuned-squad --- layout: model title: Ganda asr_wav2vec2_large_xlsr_luganda TFWav2Vec2ForCTC from lucio author: John Snow Labs name: asr_wav2vec2_large_xlsr_luganda date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_luganda` is a Ganda model originally trained by lucio. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_luganda_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_luganda_lg_4.2.0_3.0_1664036718968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_luganda_lg_4.2.0_3.0_1664036718968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_luganda", "lg")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_luganda", "lg") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_luganda| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lg| |Size:|1.2 GB| --- layout: model title: English image_classifier_vit_base_patch16_224_in21k_bantai_v1 ViTForImageClassification from AykeeSalazar author: John Snow Labs name: image_classifier_vit_base_patch16_224_in21k_bantai_v1 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_in21k_bantai_v1` is a English model originally trained by AykeeSalazar. ## Predicted Entities `Public Smoking`, `Public-Drinking`, `ambiguous`, `non-violation` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_bantai_v1_en_4.1.0_3.0_1660168998995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_in21k_bantai_v1_en_4.1.0_3.0_1660168998995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_in21k_bantai_v1", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_in21k_bantai_v1", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_in21k_bantai_v1| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Conditions precedent Clause Binary Classifier (md) author: John Snow Labs name: legclf_conditions_precedent_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `conditions-precedent` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `conditions-precedent` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_conditions_precedent_md_en_1.0.0_3.0_1673460306136.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_conditions_precedent_md_en_1.0.0_3.0_1673460306136.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_conditions_precedent_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[conditions-precedent]| |[other]| |[other]| |[conditions-precedent]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_conditions_precedent_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support other 1.00 0.97 0.99 39 term-of-agreement 0.97 1.00 0.98 32 accuracy 0.99 71 macro avg 0.98 0.99 0.99 71 weighted avg 0.99 0.99 0.99 71 ``` --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_ManuERT_for_xqua date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ManuERT-for-xqua` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_ManuERT_for_xqua_en_4.0.0_3.0_1654178819404.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_ManuERT_for_xqua_en_4.0.0_3.0_1654178819404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_ManuERT_for_xqua","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_ManuERT_for_xqua","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_ManuERT_for_xqua| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/ManuERT-for-xqua --- layout: model title: English Deberta Embeddings model (from domenicrosati) author: John Snow Labs name: deberta_embeddings_v3_large_dapt_scientific_papers_pubmed date: 2023-03-13 tags: [deberta, open_source, deberta_embeddings, debertav2formaskedlm, en, tensorflow] task: Embeddings language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DeBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DebertaEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deberta-v3-large-dapt-scientific-papers-pubmed` is a English model originally trained by `domenicrosati`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_en_4.3.1_3.0_1678700874628.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/deberta_embeddings_v3_large_dapt_scientific_papers_pubmed_en_4.3.1_3.0_1678700874628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_v3_large_dapt_scientific_papers_pubmed","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_v3_large_dapt_scientific_papers_pubmed","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_v3_large_dapt_scientific_papers_pubmed| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|false| ## References https://huggingface.co/domenicrosati/deberta-v3-large-dapt-scientific-papers-pubmed --- layout: model title: English asr_wav2vec2_large_robust_libri_960h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_large_robust_libri_960h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_libri_960h` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_robust_libri_960h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_libri_960h_en_4.2.0_3.0_1664039426765.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_libri_960h_en_4.2.0_3.0_1664039426765.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_robust_libri_960h", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_robust_libri_960h", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_robust_libri_960h| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|757.5 MB| --- layout: model title: Translate Sranan Tongo to English Pipeline author: John Snow Labs name: translate_srn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, srn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `srn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_srn_en_xx_2.7.0_2.4_1609689416373.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_srn_en_xx_2.7.0_2.4_1609689416373.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_srn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_srn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.srn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_srn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spell Checker in English Text author: John Snow Labs name: check_spelling_dl date: 2021-03-23 tags: [en, open_source] supported: true task: Spell Check language: en nav_key: models edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Spell Checker is a sequence-to-sequence pipeline that detects and corrects spelling errors in your input text. It's based on Levenshtein Automaton for generating candidate corrections and a Neural Language Model for ranking corrections. You can download the pretrained pipeline that comes ready to use. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/check_spelling_dl_en_2.7.5_2.4_1616498835957.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/check_spelling_dl_en_2.7.5_2.4_1616498835957.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In order to use this pretrained pipeline, you need to just provide the text to be checked.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('check_spelling_dl', lang='en') result = pipeline.fullAnnotate("During the summer we have the hottest ueather. I have a black ueather jacket, so nice.I intrduce you to my sister, she is called ueather.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("check_spelling_dl", lang = "en") val result = pipeline.fullAnnotate("During the summer we have the hottest ueather. I have a black ueather jacket, so nice.I intrduce you to my sister, she is called ueather.") ``` {:.nlu-block} ```python import nlu nlu.load("en.spell").predict("""During the summer we have the hottest ueather. I have a black ueather jacket, so nice.I intrduce you to my sister, she is called ueather.""") ```

## Results ```bash [('During', 'During'), ('the', 'the'), ('summer', 'summer'), ('we', 'we'), ('have', 'have'), ('the', 'the'), ('hottest', 'hottest'), ('ueather', 'weather'), ('.', '.'), ('I', 'I'), ('have', 'have'), ('a', 'a'), ('black', 'black'), ('ueather', 'leather'), ('jacket', 'jacket'), (',', ','), ('so', 'so'), ('nice', 'nice'), ('.', '.'), ('I', 'I'), ('intrduce', 'introduce'), ('you', 'you'), ('to', 'to'), ('my', 'my'), ('sister', 'sister'), (',', ','), ('she', 'she'), ('is', 'is'), ('called', 'called'), ('ueather', 'Heather'), ('.', '.')] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|check_spelling_dl| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Language:|en| ## Included Models `SentenceDetectorDLModel` `ContextSpellCheckerModel` --- layout: model title: English BertForQuestionAnswering model (from howey) author: John Snow Labs name: bert_qa_bert_base_uncased_squad_L3 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_base_uncased_squad_L3` is a English model orginally trained by `howey`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad_L3_en_4.0.0_3.0_1654185111402.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad_L3_en_4.0.0_3.0_1654185111402.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squad_L3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squad_L3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_l3.by_howey").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squad_L3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|169.0 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/howey/bert_base_uncased_squad_L3 --- layout: model title: Spell Checking Pipeline for English author: John Snow Labs name: check_spelling date: 2021-03-26 tags: [open_source, english, check_spelling, pipeline, en] supported: true task: [Spell Check,] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The check_spelling is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SPELL_CHECKER_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SPELL_CHECKER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/check_spelling_en_3.0.0_3.0_1616772629811.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/check_spelling_en_3.0.0_3.0_1616772629811.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('check_spelling', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("check_spelling", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('').predict(text) result_df ```

## Results ```bash | | document | sentence | token | checked | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------------------------| | 0 | ['I liek to live dangertus ! '] | ['I liek to live dangertus !'] | ['I', 'liek', 'to', 'live', 'dangertus', '!'] | ['I', 'like', 'to', 'live', 'dangerous', '!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|check_spelling| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - NorvigSweetingModel --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381_de_4.2.0_3.0_1664112699216.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381_de_4.2.0_3.0_1664112699216.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_0_austria_10_s381| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Russian T5ForConditionalGeneration Base Cased model (from 0x7194633) author: John Snow Labs name: t5_keyt5_base date: 2023-01-30 tags: [ru, open_source, t5, tensorflow] task: Text Generation language: ru edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyt5-base` is a Russian model originally trained by `0x7194633`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_keyt5_base_ru_4.3.0_3.0_1675104774932.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_keyt5_base_ru_4.3.0_3.0_1675104774932.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_keyt5_base","ru") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_keyt5_base","ru") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_keyt5_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ru| |Size:|927.4 MB| ## References - https://huggingface.co/0x7194633/keyt5-base - https://github.com/0x7o/text2keywords - https://github.com/0x7o/text2keywords - https://github.com/0x7o/text2keywords - https://github.com/0x7o/text2keywords - https://colab.research.google.com/github/0x7o/text2keywords/blob/main/example/keyT5_use.ipynb - https://colab.research.google.com/github/0x7o/text2keywords/blob/main/example/keyT5_train.ipynb --- layout: model title: English image_classifier_vit_violation_classification_bantai__v100ep ViTForImageClassification from AykeeSalazar author: John Snow Labs name: image_classifier_vit_violation_classification_bantai__v100ep date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_violation_classification_bantai__v100ep` is a English model originally trained by AykeeSalazar. ## Predicted Entities `Public Smoking`, `Public-Drinking`, `ambiguous`, `non-violation` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_violation_classification_bantai__v100ep_en_4.1.0_3.0_1660169126596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_violation_classification_bantai__v100ep_en_4.1.0_3.0_1660169126596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_violation_classification_bantai__v100ep", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_violation_classification_bantai__v100ep", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_violation_classification_bantai__v100ep| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-512-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6_en_4.0.0_3.0_1657185261800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6_en_4.0.0_3.0_1657185261800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_512_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-512-finetuned-squad-seed-6 --- layout: model title: Stopwords Remover for Ukrainian language (467 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, uk, open_source] task: Stop Words Removal language: uk edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_uk_3.4.1_3.0_1646673071386.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_uk_3.4.1_3.0_1646673071386.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[] | +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|uk| |Size:|3.1 KB| --- layout: model title: English RobertaForQuestionAnswering (from Salesforce) author: John Snow Labs name: roberta_qa_qaconv_roberta_large_squad2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qaconv-roberta-large-squad2` is a English model originally trained by `Salesforce`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_qaconv_roberta_large_squad2_en_4.0.0_3.0_1655729360534.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_qaconv_roberta_large_squad2_en_4.0.0_3.0_1655729360534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_qaconv_roberta_large_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_qaconv_roberta_large_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.large.by_Salesforce").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_qaconv_roberta_large_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Salesforce/qaconv-roberta-large-squad2 --- layout: model title: Legal Employment agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_employment_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_employment_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class employment-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `employment-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_employment_agreement_en_1.0.0_3.0_1666620976596.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_employment_agreement_en_1.0.0_3.0_1666620976596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_employment_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[employment-agreement]| |[other]| |[other]| |[employment-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_employment_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|20.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support employment-agreement 0.96 1.00 0.98 24 other 1.00 0.98 0.99 62 accuracy - - 0.99 86 macro-avg 0.98 0.99 0.99 86 weighted-avg 0.99 0.99 0.99 86 ``` --- layout: model title: Legal Loan Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_loan_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, loan, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_loan_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `loan-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `loan-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_loan_agreement_bert_en_1.0.0_3.0_1669316061148.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_loan_agreement_bert_en_1.0.0_3.0_1669316061148.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_loan_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[loan-agreement]| |[other]| |[other]| |[loan-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_loan_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support loan-agreement 0.72 0.97 0.82 29 other 0.97 0.73 0.83 41 accuracy - - 0.83 70 macro-avg 0.84 0.85 0.83 70 weighted-avg 0.86 0.83 0.83 70 ``` --- layout: model title: Translate Wolaytta to English Pipeline author: John Snow Labs name: translate_wal_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, wal, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `wal` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_wal_en_xx_2.7.0_2.4_1609691928141.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_wal_en_xx_2.7.0_2.4_1609691928141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_wal_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_wal_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.wal.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_wal_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl32 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl32` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl32_en_4.3.0_3.0_1675122269324.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl32_en_4.3.0_3.0_1675122269324.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl32","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl32","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl32| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|514.9 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl32 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Translate Gilbertese to English Pipeline author: John Snow Labs name: translate_gil_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, gil, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `gil` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_gil_en_xx_2.7.0_2.4_1609690078155.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_gil_en_xx_2.7.0_2.4_1609690078155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gil_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gil_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gil.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gil_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from rahulkuruvilla) A Version author: John Snow Labs name: distilbert_qa_COVID_DistilBERTa date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `COVID-DistilBERTa` is a English model originally trained by `rahulkuruvilla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_COVID_DistilBERTa_en_4.0.0_3.0_1654722884791.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_COVID_DistilBERTa_en_4.0.0_3.0_1654722884791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_COVID_DistilBERTa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_COVID_DistilBERTa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.covid.distil_bert.a.by_rahulkuruvilla").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_COVID_DistilBERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rahulkuruvilla/COVID-DistilBERTa --- layout: model title: Word2Vec Embeddings in Danish (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, da, open_source] task: Embeddings language: da edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_da_3.4.1_3.0_1647293782911.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_da_3.4.1_3.0_1647293782911.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_squad_2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_squad_2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.albert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_squad_2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SS8/albert_squad_2.0 --- layout: model title: English asr_wav2vec2_large_xlsr_nahuatl TFWav2Vec2ForCTC from tyoc213 author: John Snow Labs name: asr_wav2vec2_large_xlsr_nahuatl date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_nahuatl` is a English model originally trained by tyoc213. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_nahuatl_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_nahuatl_en_4.2.0_3.0_1664110129570.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_nahuatl_en_4.2.0_3.0_1664110129570.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_nahuatl", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_nahuatl", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_nahuatl| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from anurag0077) author: John Snow Labs name: distilbert_qa_anurag0077_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `anurag0077`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_anurag0077_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769885692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_anurag0077_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769885692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anurag0077_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anurag0077_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_anurag0077_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anurag0077/distilbert-base-uncased-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering Cased model (from skandaonsolve) author: John Snow Labs name: roberta_qa_finetuned_city date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-city` is a English model originally trained by `skandaonsolve`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_city_en_4.3.0_3.0_1674220204421.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_city_en_4.3.0_3.0_1674220204421.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_city","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_city","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_city| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skandaonsolve/roberta-finetuned-city --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_ff12000 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-ff12000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff12000_en_4.3.0_3.0_1675120874161.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff12000_en_4.3.0_3.0_1675120874161.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_ff12000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_ff12000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_ff12000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|388.5 MB| ## References - https://huggingface.co/google/t5-efficient-small-ff12000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Pipeline to Detect Time-related Terminology author: John Snow Labs name: roberta_token_classifier_timex_semeval_pipeline date: 2022-03-19 tags: [timex, semeval, ner, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_token_classifier_timex_semeval](https://nlp.johnsnowlabs.com/2021/12/28/roberta_token_classifier_timex_semeval_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_TIMEX_SEMEVAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_timex_semeval_pipeline_en_3.4.1_3.0_1647699181926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_timex_semeval_pipeline_en_3.4.1_3.0_1647699181926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python timex_pipeline = PretrainedPipeline("roberta_token_classifier_timex_semeval_pipeline", lang = "en") timex_pipeline.annotate("Model training was started at 22:12C and it took 3 days from Tuesday to Friday.") ``` ```scala val timex_pipeline = new PretrainedPipeline("roberta_token_classifier_timex_semeval_pipeline", lang = "en") timex_pipeline.annotate("Model training was started at 22:12C and it took 3 days from Tuesday to Friday.") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") text = '''Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") val text = "Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.cellular_pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------------------------------|--------:|------:|:------------|-------------:| | 0 | intracellular signaling proteins | 27 | 58 | protein | 0.999477 | | 1 | human T-cell leukemia virus type 1 promoter | 130 | 172 | DNA | 0.999333 | | 2 | Tax | 186 | 188 | protein | 0.999844 | | 3 | Tax-responsive element 1 | 193 | 216 | DNA | 0.999461 | | 4 | cyclic AMP-responsive members | 237 | 265 | protein | 0.994097 | | 5 | CREB/ATF family | 274 | 288 | protein | 0.995731 | | 6 | transcription factors | 293 | 313 | protein | 0.993303 | | 7 | Tax | 389 | 391 | protein | 0.999503 | | 8 | human T-cell leukemia virus type 1 | 396 | 429 | DNA | 0.93039 | | 9 | Tax-responsive element 1 | 431 | 454 | DNA | 0.99246 | | 10 | TRE-1 | 457 | 461 | DNA | 0.999713 | | 11 | lacZ gene | 582 | 590 | DNA | 0.998996 | | 12 | CYC1 promoter | 617 | 629 | DNA | 0.99915 | | 13 | TRE-1 | 663 | 667 | DNA | 0.99943 | | 14 | cyclic AMP response element-binding protein | 695 | 737 | protein | 0.999852 | | 15 | CREB | 740 | 743 | protein | 0.999918 | | 16 | CREB | 749 | 752 | protein | 0.999922 | | 17 | GAL4 activation domain | 767 | 788 | protein | 0.931828 | | 18 | GAD | 791 | 793 | protein | 0.999684 | | 19 | reporter gene | 848 | 860 | DNA | 0.998856 | | 20 | Tax | 863 | 865 | protein | 0.999717 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Chinese BertForMaskedLM Mini Cased model (from hfl) author: John Snow Labs name: bert_embeddings_minirbt_h288 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `minirbt-h288` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_minirbt_h288_zh_4.2.4_3.0_1670022638878.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_minirbt_h288_zh_4.2.4_3.0_1670022638878.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_minirbt_h288","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_minirbt_h288","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_minirbt_h288| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|46.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/minirbt-h288 - https://github.com/iflytek/MiniRBT - https://github.com/ymcui/LERT - https://github.com/ymcui/PERT - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/iflytek/HFL-Anthology --- layout: model title: BERT Sentence Embeddings (Large Cased) author: John Snow Labs name: sent_bert_large_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus. The details are described in the paper "[BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_large_cased_en_2.6.0_2.4_1598346401930.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_large_cased_en_2.6.0_2.4_1598346401930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_bert_large_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_bert_large_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer", "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.bert_large_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_sentence_bert_large_cased_embeddings I [[-0.6228358149528503, -0.3453695774078369, 0.... love [[-0.6228358149528503, -0.3453695774078369, 0.... NLP [[-0.6228358149528503, -0.3453695774078369, 0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_large_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/bert_cased_L-24_H-1024_A-16/1](https://tfhub.dev/google/bert_cased_L-24_H-1024_A-16/1) --- layout: model title: Pipeline to Mapping ICDO Codes with Their Corresponding SNOMED Codes author: John Snow Labs name: icdo_snomed_mapping date: 2023-03-29 tags: [en, licensed, clinical, resolver, pipeline, chunk_mapping, icdo, snomed] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `icdo_snomed_mapper` model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icdo_snomed_mapping_en_4.3.2_3.2_1680120340177.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icdo_snomed_mapping_en_4.3.2_3.2_1680120340177.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icdo_snomed_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(8120/1 8170/3 8380/3) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icdo_snomed_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(8120/1 8170/3 8380/3) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.icdo_to_snomed.pipe").predict("""Put your text here.""") ```

## Results ```bash | | icdo_code | snomed_code | |---:|:-------------------------|:-------------------------------| | 0 | 8120/1 | 8170/3 | 8380/3 | 45083001 | 25370001 | 30289006 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icdo_snomed_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|133.3 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Fast Neural Machine Translation Model from Walloon to English author: John Snow Labs name: opus_mt_wa_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, wa, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `wa` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_wa_en_xx_2.7.0_2.4_1609166413374.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_wa_en_xx_2.7.0_2.4_1609166413374.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_wa_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Оcr base v2 for printed text author: John Snow Labs name: ocr_base_printed_v2 date: 2023-01-17 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 4.2.4 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Ocr base printed model v2 for recognise printed text based on a TrOCR model pretrained with printed datasets. It is an Ocr base model for recognise handwritten text based on TrOcr architecture. The TrOCR model was proposed in TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. TrOCR consists of an image Transformer encoder and an autoregressive text Transformer decoder to perform optical character recognition (OCR). The abstract from the paper is the following: Text recognition is a long-standing research problem for document digitalization. Existing approaches for text recognition are usually built based on CNN for image understanding and RNN for char-level text generation. In addition, another language model is usually needed to improve the overall accuracy as a post-processing step. In this paper, we propose an end-to-end text recognition approach with pre-trained image Transformer and text Transformer models, namely TrOCR, which leverages the Transformer architecture for both image understanding and wordpiece-level text generation. The TrOCR model is simple but effective, and can be pre-trained with large-scale synthetic data and fine-tuned with human-labeled datasets. Experiments show that the TrOCR model outperforms the current state-of-the-art models on both printed and handwritten text recognition tasks. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/RECOGNIZE_PRINTED/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/jupyter/Cards/SparkOcrImageToTextPrinted_V2.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_base_printed_v2_en_4.2.2_3.0_1670623909000.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_base_printed_v2_en_4.2.2_3.0_1670623909000.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) text_detector = ImageTextDetectorV2 \ .pretrained("image_text_detector_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("text_regions") \ .setWithRefiner(True) \ .setSizeThreshold(-1) \ .setLinkThreshold(0.3) \ .setWidth(500) ocr = ImageToTextV2Opt.pretrained("ocr_base_printed_v2", "en", "clinical/ocr") \ .setInputCols(["image", "text_regions"]) \ .setGroupImages(True) \ .setOutputCol("text") \ .setRegionsColumn("text_regions") draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("text_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.green) \ .setRotated(True) pipeline = PipelineModel(stages=[ binary_to_image, text_detector, ocr, draw_regions ]) imagePath = pkg_resources.resource_filename('sparkocr', 'resources/ocr/images/check.jpg') image_df = spark.read.format("binaryFile").load(imagePath) result = pipeline.transform(image_df).cache() ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val text_detector = ImageTextDetectorV2 .pretrained("image_text_detector_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("text_regions") .setWithRefiner(True) .setSizeThreshold(-1) .setLinkThreshold(0.3) .setWidth(500) val ocr = ImageToTextV2Opt .pretrained("ocr_base_printed_v2", "en", "clinical/ocr") .setInputCols(Array("image", "text_regions")) .setGroupImages(True) .setOutputCol("text") .setRegionsColumn("text_regions") val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("text_regions") .setOutputCol("image_with_regions") .setRectColor(Color.green) .setRotated(True) val pipeline = new PipelineModel().setStages(Array(binary_to_image, text_detector, ocr, draw_regions)) val imagePath = pkg_resources.resource_filename("sparkocr", "resources/ocr/images/check.jpg") val image_df = spark.read.format("binaryFile").load(imagePath) val result = pipeline.transform(image_df).cache() ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image2.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image2_out3.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash STARBUCKS STORE #10208 11302 EUCLID AVENUE CLEVELAND, OH (216) 229-0749 CHK 664290 12/07/2014 06:43 PM 1912003- DRAWER: 2. REG: 2 VT PEP MOCHA SBUX CARD : 4.95 XXXXXXXXXXXX3228 SUBTOTAL: $4.95 TOTAL @ 6.95 CHANGE DUE $0.00 ................ CCHECK CLOSED 12/07/2014 06:43 PM SBUX CARD X3228 NEW BALANCE: 37.45 CARD IS REGISTERED ``` ## Model Information {:.table-model} |---|---| |Model Name:|ocr_base_printed_v2| |Type:|ocr| |Compatibility:|Visual NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Legal Merger Clause Binary Classifier author: John Snow Labs name: legclf_merger_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `merger` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `merger` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_merger_clause_en_1.0.0_3.2_1660122654083.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_merger_clause_en_1.0.0_3.2_1660122654083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_merger_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[merger]| |[other]| |[other]| |[merger]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_merger_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support merger 0.83 0.90 0.86 39 other 0.96 0.94 0.95 109 accuracy - - 0.93 148 macro-avg 0.90 0.92 0.91 148 weighted-avg 0.93 0.93 0.93 148 ``` --- layout: model title: Detect Assertion Status from Oncology Entities author: John Snow Labs name: assertion_oncology_wip date: 2022-10-11 tags: [licensed, clinical, oncology, en, assertion] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of entities related to oncology (including diagnoses, therapies and tests). ## Predicted Entities `Absent`, `Family`, `Hypothetical`, `Past`, `Possible`, `Present` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_wip_en_4.0.0_3.0_1665519630515.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_wip_en_4.0.0_3.0_1665519630515.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------------|:---------------|:------------| | lung cancer | Cancer_Dx | Possible | | cancers | Cancer_Dx | Family | | biopsy | Pathology_Test | Past | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Absent 0.83 0.79 0.81 368.0 Family 0.80 0.80 0.80 40.0 Hypothetical 0.65 0.57 0.61 229.0 Past 0.90 0.91 0.91 2124.0 Possible 0.64 0.61 0.63 85.0 Present 0.87 0.88 0.88 2121.0 macro-avg 0.78 0.76 0.77 4967.0 weighted-avg 0.87 0.87 0.87 4967.0 ``` --- layout: model title: Sentiment Analysis for Thai (sentiment_jager_use) author: John Snow Labs name: sentiment_jager_use date: 2021-01-14 task: Sentiment Analysis language: th edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [sentiment, th, open_source] supported: true annotator: SentimentDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Analyze sentiment in reviews by classifying them as `positive` and `negative`. When the sentiment probability is below a customizable threshold (default to `0.6`) then resulting document will be labeled as `neutral`. This model is trained using the multilingual `UniversalSentenceEncoder` sentence embeddings, and uses DL approach to classify the sentiments. ## Predicted Entities `positive`, `negative`, `neutral` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/5.Text_Classification_with_ClassifierDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentiment_jager_use_th_2.7.1_2.4_1610586390122.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentiment_jager_use_th_2.7.1_2.4_1610586390122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use in the pipeline with the pretrained multi-language `UniversalSentenceEncoder` annotator `tfhub_use_multi_lg`.

## Results ```bash +-------------------------------------+----------+ |text |result | +-------------------------------------+----------+ |เเพ้ตอนnctโผล่มาตลอดเลยค่ะเเอด5555555 |[positive] | +-------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentiment_jager_use| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[sentiment]| |Language:|th| ## Data Source The model was trained on the custom corpus from [Jager V3](https://github.com/JagerV3/sentiment_analysis_thai). ## Benchmarking ```bash | sentiment | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | negative | 0.94 | 0.99 | 0.96 | 82 | | positive | 0.97 | 0.87 | 0.92 | 38 | | accuracy | | | 0.95 | 120 | | macro avg | 0.96 | 0.93 | 0.94 | 120 | | weighted avg | 0.95 | 0.95 | 0.95 | 120 | ``` --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_scrambled_squad_5_new date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-5-new` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_5_new_en_4.0.0_3.0_1655734213398.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_5_new_en_4.0.0_3.0_1655734213398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_5_new","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_5_new","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_scrambled_sq.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_scrambled_squad_5_new| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-5-new --- layout: model title: Finnish asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP TFWav2Vec2ForCTC from Finnish-NLP author: John Snow Labs name: asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP` is a Finnish model originally trained by Finnish-NLP. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP_fi_4.2.0_3.0_1664039192288.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP_fi_4.2.0_3.0_1664039192288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_Finnish_NLP| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|3.6 GB| --- layout: model title: Swedish asr_Wav2Vec2_large_xlsr_welsh TFWav2Vec2ForCTC from Srulikbdd author: John Snow Labs name: pipeline_asr_Wav2Vec2_large_xlsr_welsh date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_large_xlsr_welsh` is a Swedish model originally trained by Srulikbdd. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Wav2Vec2_large_xlsr_welsh_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Wav2Vec2_large_xlsr_welsh_sv_4.2.0_3.0_1664115162978.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Wav2Vec2_large_xlsr_welsh_sv_4.2.0_3.0_1664115162978.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Wav2Vec2_large_xlsr_welsh', lang = 'sv') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Wav2Vec2_large_xlsr_welsh", lang = "sv") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Wav2Vec2_large_xlsr_welsh| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|sv| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline to Detect Genetic Cancer Entities author: John Snow Labs name: ner_cancer_genetics_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_cancer_genetics](https://nlp.johnsnowlabs.com/2021/03/31/ner_cancer_genetics_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_pipeline_en_3.4.1_3.0_1647870755233.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cancer_genetics_pipeline_en_3.4.1_3.0_1647870755233.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_cancer_genetics_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.") ``` ```scala val pipeline = new PretrainedPipeline("ner_cancer_genetics_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cancer_genetics.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""") ```

## Results ```bash +-----------------------------------------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------------------------------------+---------+ |human KCNJ9 |protein | |Kir 3.3 |protein | |GIRK3 |protein | |G-protein-activated inwardly rectifying potassium (GIRK) channel family|protein | |KCNJ9 locus |DNA | |chromosome 1q21-23 |DNA | |coding exons |DNA | |identified14 single nucleotide polymorphisms |DNA | |SNPs), |DNA | |KCNJ9 gene |DNA | |KCNJ9 protein |protein | |locus |DNA | +-----------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cancer_genetics_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English image_classifier_vit_dog_breed_classifier ViTForImageClassification from skyau author: John Snow Labs name: image_classifier_vit_dog_breed_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_dog_breed_classifier` is a English model originally trained by skyau. ## Predicted Entities `whippet`, `cairn`, `Old_English_sheepdog`, `Rottweiler`, `American_Staffordshire_terrier`, `Blenheim_spaniel`, `Leonberg`, `bluetick`, `Yorkshire_terrier`, `African_hunting_dog`, `Doberman`, `Appenzeller`, `Boston_bull`, `German_shepherd`, `kuvasz`, `standard_poodle`, `Chesapeake_Bay_retriever`, `toy_terrier`, `Australian_terrier`, `Dandie_Dinmont`, `Brittany_spaniel`, `basenji`, `Newfoundland`, `Airedale`, `giant_schnauzer`, `Bouvier_des_Flandres`, `golden_retriever`, `Welsh_springer_spaniel`, `Pekinese`, `West_Highland_white_terrier`, `briard`, `Gordon_setter`, `Border_collie`, `Pomeranian`, `Scotch_terrier`, `malamute`, `EntleBucher`, `toy_poodle`, `Mexican_hairless`, `clumber`, `Scottish_deerhound`, `curly-coated_retriever`, `Bedlington_terrier`, `soft-coated_wheaten_terrier`, `Irish_setter`, `Lhasa`, `bloodhound`, `French_bulldog`, `standard_schnauzer`, `Chihuahua`, `borzoi`, `Sealyham_terrier`, `malinois`, `Norwegian_elkhound`, `Staffordshire_bullterrier`, `bull_mastiff`, `Ibizan_hound`, `komondor`, `Kerry_blue_terrier`, `Saint_Bernard`, `basset`, `Eskimo_dog`, `Sussex_spaniel`, `English_springer`, `flat-coated_retriever`, `cocker_spaniel`, `Tibetan_terrier`, `Shih-Tzu`, `beagle`, `silky_terrier`, `Saluki`, `vizsla`, `pug`, `Shetland_sheepdog`, `Maltese_dog`, `Norwich_terrier`, `kelpie`, `Italian_greyhound`, `Walker_hound`, `Greater_Swiss_Mountain_dog`, `miniature_schnauzer`, `Great_Pyrenees`, `Tibetan_mastiff`, `collie`, `Siberian_husky`, `Bernese_mountain_dog`, `Irish_wolfhound`, `chow`, `boxer`, `Great_Dane`, `dingo`, `Japanese_spaniel`, `Rhodesian_ridgeback`, `Border_terrier`, `Afghan_hound`, `Irish_water_spaniel`, `black-and-tan_coonhound`, `redbone`, `Norfolk_terrier`, `affenpinscher`, `Brabancon_griffon`, `miniature_pinscher`, `Labrador_retriever`, `Lakeland_terrier`, `groenendael`, `schipperke`, `papillon`, `wire-haired_fox_terrier`, `Cardigan`, `English_foxhound`, `Pembroke`, `dhole`, `German_short-haired_pointer`, `miniature_poodle`, `Irish_terrier`, `Weimaraner`, `otterhound`, `English_setter`, `Samoyed`, `keeshond` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_breed_classifier_en_4.1.0_3.0_1660167173151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_breed_classifier_en_4.1.0_3.0_1660167173151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_dog_breed_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_dog_breed_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_dog_breed_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.2 MB| --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1655733311668.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1655733311668.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_64d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|419.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-10 --- layout: model title: Pipeline to Detect Diseases in Medical Text author: John Snow Labs name: bert_token_classifier_ner_ncbi_disease_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_ncbi_disease](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_ncbi_disease_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ncbi_disease_pipeline_en_4.3.0_3.2_1679303325122.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ncbi_disease_pipeline_en_4.3.0_3.2_1679303325122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_ncbi_disease_pipeline", "en", "clinical/models") text = '''Kniest dysplasia is a moderately severe type II collagenopathy, characterized by short trunk and limbs, kyphoscoliosis, midface hypoplasia, severe myopia, and hearing loss.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_ncbi_disease_pipeline", "en", "clinical/models") val text = "Kniest dysplasia is a moderately severe type II collagenopathy, characterized by short trunk and limbs, kyphoscoliosis, midface hypoplasia, severe myopia, and hearing loss." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------|--------:|------:|:------------|-------------:| | 0 | Kniest dysplasia | 0 | 15 | Disease | 0.999886 | | 1 | type II collagenopathy | 40 | 61 | Disease | 0.999934 | | 2 | kyphoscoliosis | 104 | 117 | Disease | 0.99994 | | 3 | midface hypoplasia | 120 | 137 | Disease | 0.999911 | | 4 | myopia | 147 | 152 | Disease | 0.999894 | | 5 | hearing loss | 159 | 170 | Disease | 0.999351 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_ncbi_disease_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-4_H-768_A-12_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna_en_4.0.0_3.0_1654185363690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna_en_4.0.0_3.0_1654185363690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.bert.uncased_4l_768d_a12a_768d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_768_A_12_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|195.3 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-768_A-12_squad2_covid-qna --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465523 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465523` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465523_en_4.0.0_3.0_1655986981491.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465523_en_4.0.0_3.0_1655986981491.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465523","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465523","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465523.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465523| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465523 --- layout: model title: Stop Words Cleaner for Hausa author: John Snow Labs name: stopwords_ha date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: ha edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, ha] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_ha_ha_2.5.4_2.4_1594742441392.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_ha_ha_2.5.4_2.4_1594742441392.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_ha", "ha") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Ban da kasancewarsa sarkin arewa, John Snow likita ne na Ingilishi kuma jagora a ci gaban maganin sa barci da tsabtar lafiya.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_ha", "ha") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Ban da kasancewarsa sarkin arewa, John Snow likita ne na Ingilishi kuma jagora a ci gaban maganin sa barci da tsabtar lafiya.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Ban da kasancewarsa sarkin arewa, John Snow likita ne na Ingilishi kuma jagora a ci gaban maganin sa barci da tsabtar lafiya."""] stopword_df = nlu.load('ha.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=7, end=18, result='kasancewarsa', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=25, result='sarkin', metadata={'sentence': '0'}), Row(annotatorType='token', begin=27, end=31, result='arewa', metadata={'sentence': '0'}), Row(annotatorType='token', begin=32, end=32, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=34, end=37, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_ha| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ha| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0_en_4.3.0_3.0_1674214651872.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0_en_4.3.0_3.0_1674214651872.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_256_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|427.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-0 --- layout: model title: Pipeline to Mapping ICDO Codes with Their Corresponding SNOMED Codes author: John Snow Labs name: icdo_snomed_mapping date: 2022-06-27 tags: [icdo, snomed, pipeline, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `icdo_snomed_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icdo_snomed_mapping_en_3.5.3_3.0_1656364275328.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icdo_snomed_mapping_en_3.5.3_3.0_1656364275328.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("icdo_snomed_mapping", "en", "clinical/models") result= pipeline.fullAnnotate("8120/1 8170/3 8380/3") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= new PretrainedPipeline("icdo_snomed_mapping", "en", "clinical/models") val result= pipeline.fullAnnotate("8120/1 8170/3 8380/3") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.icdo_to_snomed.pipe").predict("""8120/1 8170/3 8380/3""") ```

## Results ```bash | | icdo_code | snomed_code | |---:|:-------------------------|:-------------------------------| | 0 | 8120/1 | 8170/3 | 8380/3 | 45083001 | 25370001 | 30289006 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icdo_snomed_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|133.2 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Chinese BERT with Whole Word Masking author: John Snow Labs name: chinese_bert_wwm date: 2021-05-20 tags: [chinese, zh, embeddings, bert, open_source] task: Embeddings language: zh edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description **[Pre-Training with Whole Word Masking for Chinese BERT](https://arxiv.org/abs/1906.08101)** Yiming Cui, Wanxiang Che, Ting Liu, Bing Qin, Ziqing Yang, Shijin Wang, Guoping Hu More resources by HFL: https://github.com/ymcui/HFL-Anthology If you find the technical report or resource is useful, please cite the following technical report in your paper. - Primary: [https://arxiv.org/abs/2004.13922](https://arxiv.org/abs/2004.13922) - Secondary: [https://arxiv.org/abs/1906.08101](https://arxiv.org/abs/1906.08101) {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/chinese_bert_wwm_zh_3.1.0_2.4_1621511963425.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/chinese_bert_wwm_zh_3.1.0_2.4_1621511963425.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("chinese_bert_wwm", "zh") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("chinese_bert_wwm", "zh") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("zh.embed.bert.wwm").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chinese_bert_wwm| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|zh| |Case sensitive:|true| ## Data Source [https://huggingface.co/hfl/chinese-bert-wwm](https://huggingface.co/hfl/chinese-bert-wwm) ## Benchmarking ```bash - BERTGoogle BERT-wwm BERT-wwm-ext RoBERTa-wwm-ext RoBERTa-wwm-ext-large Masking WordPiece WWM[1] WWM WWM WWM Type base base base base large Data Source wiki wiki wiki+ext[2] wiki+ext wiki+ext Training Tokens # 0.4B 0.4B 5.4B 5.4B 5.4B Device TPU Pod v2 TPU v3 TPU v3 TPU v3 TPU Pod v3-32[3] Training Steps ? 100KMAX128 +100KMAX512 1MMAX128 +400KMAX512 1MMAX512 2MMAX512 Batch Size ? 2,560 / 384 2,560 / 384 384 512 Optimizer AdamW LAMB LAMB AdamW AdamW Vocabulary 21,128 ~BERT[4] ~BERT ~BERT ~BERT Init Checkpoint Random Init ~BERT ~BERT ~BERT Random Init ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_ff6000 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-ff6000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_ff6000_en_4.3.0_3.0_1675112144975.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_ff6000_en_4.3.0_3.0_1675112144975.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_ff6000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_ff6000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_ff6000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|692.3 MB| ## References - https://huggingface.co/google/t5-efficient-base-ff6000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Cusip Numbers Clause Binary Classifier author: John Snow Labs name: legclf_cusip_numbers_clause date: 2023-01-29 tags: [en, legal, classification, cusip, numbers, clauses, cusip_numbers, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `cusip-numbers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `cusip-numbers`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cusip_numbers_clause_en_1.0.0_3.0_1674994284758.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cusip_numbers_clause_en_1.0.0_3.0_1674994284758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_cusip_numbers_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[cusip-numbers]| |[other]| |[other]| |[cusip-numbers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cusip_numbers_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support cusip-numbers 0.93 0.96 0.95 28 other 0.97 0.95 0.96 37 accuracy - - 0.95 65 macro-avg 0.95 0.96 0.95 65 weighted-avg 0.95 0.95 0.95 65 ``` --- layout: model title: Detect Chemical Compounds and Genes (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_chemprot date: 2022-01-06 tags: [berfortokenclassification, chemprot, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect chemical compounds and genes in the medical text using the pretrained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `CHEMICAL`, `GENE-Y`, `GENE-N` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_en_3.3.4_2.4_1641471274375.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_en_3.3.4_2.4_1641471274375.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_mh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_mh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.mh.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_mh_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect PHI for Deidentification (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_deid date: 2021-09-13 tags: [en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.2.0 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deidentification NER is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 23 entities. This ner model is trained with a combination of the i2b2 train set and a re-augmented version of i2b2 train set using `BertForTokenClassification` We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `MEDICALRECORD`, `ORGANIZATION`, `DOCTOR`, `USERNAME`, `PROFESSION`, `HEALTHPLAN`, `URL`, `CITY`, `DATE`, `LOCATION-OTHER`, `STATE`, `PATIENT`, `DEVICE`, `COUNTRY`, `ZIP`, `PHONE`, `HOSPITAL`, `EMAIL`, `IDNUM`, `SREET`, `BIOID`, `FAX`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_deid_en_3.2.1_2.4_1631538493075.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_deid_en_3.2.1_2.4_1631538493075.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------+-------------+ |chunk |ner_label | +-----------------------------+-------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson, Ora |PATIENT | |7194334 |MEDICALRECORD| |Oliveira |PATIENT | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street |STREET | |302) 786-5227 |PHONE | |Brothers Coal-Mine |ORGANIZATION | +-----------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_deid| |Compatibility:|Healthcare NLP 3.2.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|128| ## Data Source A custom data set which is created from the i2b2-PHI train and the re-augmented version of the i2b2-PHI train set is used. ## Benchmarking ```bash label precision recall f1-score support B-AGE 0.92 0.80 0.86 1050 B-CITY 0.71 0.93 0.80 530 B-COUNTRY 0.94 0.72 0.82 179 B-DATE 0.99 0.99 0.99 20434 B-DEVICE 0.68 0.66 0.67 35 B-DOCTOR 0.93 0.91 0.92 3609 B-EMAIL 0.92 1.00 0.96 11 B-HOSPITAL 0.94 0.90 0.92 2225 B-IDNUM 0.88 0.64 0.74 1185 B-LOCATION-OTHER 0.83 0.60 0.70 25 B-MEDICALRECORD 0.84 0.97 0.90 2263 B-ORGANIZATION 0.92 0.68 0.79 171 B-PATIENT 0.89 0.86 0.88 2297 B-PHONE 0.90 0.96 0.93 755 B-PROFESSION 0.86 0.81 0.83 265 B-STATE 0.97 0.87 0.92 261 B-STREET 0.98 0.99 0.99 184 B-USERNAME 0.96 0.78 0.86 357 B-ZIP 0.96 0.96 0.96 444 I-CITY 0.74 0.83 0.78 138 I-DATE 0.98 0.96 0.97 955 I-DEVICE 1.00 1.00 1.00 3 I-DOCTOR 0.92 0.98 0.95 3134 I-HOSPITAL 0.95 0.92 0.93 1322 I-LOCATION-OTHER 1.00 1.00 1.00 8 I-MEDICALRECORD 1.00 0.91 0.95 23 I-ORGANIZATION 0.98 0.61 0.75 77 I-PATIENT 0.89 0.81 0.85 1281 I-PHONE 0.97 0.96 0.97 374 I-PROFESSION 0.95 0.82 0.88 232 I-STREET 0.98 0.98 0.98 391 O 1.00 1.00 1.00 585606 accuracy - - 0.99 629960 macro-avg 0.79 0.71 0.73 629960 weighted-avg 0.99 0.99 0.99 629960 ``` --- layout: model title: Clinical Deidentification Pipeline (Portuguese) author: John Snow Labs name: clinical_deidentification date: 2022-04-14 tags: [deid, deidentification, pt, licensed] task: [De-identification, Pipeline Healthcare] language: pt edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true recommended: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with `w2v_cc_300d` portuguese embeddings and can be used to deidentify PHI information from medical texts in Portuguese. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask, fake or obfuscate the following entities: `AGE`, `DATE`, `PROFESSION`, `EMAIL`, `ID`, `COUNTRY`, `STREET`, `DOCTOR`, `HOSPITAL`, `PATIENT`, `URL`, `IP`, `ORGANIZATION`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `PLATE`, `SEX` and `IPADDR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_pt_3.4.1_3.0_1649956332889.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_pt_3.4.1_3.0_1649956332889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Masked with entity labels ------------------------------ Dados do . Nome: . Apelido: . NIF: . NISS: . Endereço: . CÓDIGO POSTAL: . Dados de cuidados. Data de nascimento: . País: . Idade: anos Sexo: . Data de admissão: . Doutor: Cuéllar NºCol: . Relatório clínico do : de anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: - , 22 E-mail: . Masked with chars ------------------------------ Dados do [******]. Nome: [***]. Apelido: [*******]. NIF: [****]. NISS: [*********]. Endereço: [*********************]. CÓDIGO POSTAL: [***]. Dados de cuidados. Data de nascimento: [********]. País: [******]. Idade: ** anos Sexo: *. Data de admissão: [********]. Doutor: [*************] Cuéllar NºCol: ** ** [***]. Relatório clínico do [******]: [******] de ** anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér[**] de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: [***********] - [*****************], 22 [******] E-mail: [****************]. Masked with fixed length chars ------------------------------ Dados do ****. Nome: ****. Apelido: ****. NIF: ****. NISS: ****. Endereço: ****. CÓDIGO POSTAL: ****. Dados de cuidados. Data de nascimento: ****. País: ****. Idade: **** anos Sexo: ****. Data de admissão: ****. Doutor: **** Cuéllar NºCol: **** **** ****. Relatório clínico do ****: **** de **** anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicér**** de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: **** - ****, 22 **** E-mail: ****. Obfuscated ------------------------------ Dados do H.. Nome: Marcos Alves. Apelido: Tiago Santos. NIF: 566-445. NISS: 134544332. Endereço: Rua de Santa María, 100. CÓDIGO POSTAL: 4099. Dados de cuidados. Data de nascimento: 31/03/1946. País: Espanha. Idade: 46 anos Sexo: Mulher. Data de admissão: 06/01/2017. Doutor: Carlos Melo Cuéllar NºCol: 134544332 134544332 124 445 311. Relatório clínico do H.: M. de 46 anos, mineiro reformado, sem alergias medicamentosas conhecidas, que apresenta como história pessoal: acidente de trabalho antigo com fracturas vertebrais e das costelas; operado por doença de Dupuytren na mão direita e iliofemoral esquerda; Diabetes Mellitus tipo II, hipercolesterolemia e hiperuricemia; alcoolismo activo, fumador de 20 cigarros / dia. Foi encaminhado dos cuidados primários porque apresentou uma vez hematúria macroscópica pós-morte e depois microhaematúria persistente, com micturição normal. O exame físico mostrou um bom estado geral, com abdómen e genitália normais; o exame rectal foi compatível com adenoma de próstata de grau I/IV. A urinálise mostrou 4 glóbulos vermelhos/campo e 0-5 leucócitos/campo; o resto do sedimento estava normal. Hemograma normal; a bioquímica mostrou glicemia de 169 mg/dl e triglicérHomen de 456 mg/dl; função hepática e renal normal. PSA de 1,16 ng/ml. A citologia da urina era repetidamente desconfiada por malignidade. A radiografia simples abdominal mostra alterações degenerativas na coluna lombar e calcificações vasculares tanto no hipocôndrio como na pélvis. A ecografia urológica revelou cistos corticais simples no rim direito, uma bexiga inalterada com boa capacidade e uma próstata com 30g de peso. O IVUS mostrou normofuncionalismo renal bilateral, calcificações na silhueta renal direita e ureteres artrosados com imagens de adição no terço superior de ambos os ureteres, relacionadas com pseudodiverticulose ureteral. O cistograma mostra uma bexiga com boa capacidade, mas com paredes trabeculadas em relação à bexiga de stress. A tomografia computorizada abdominal é normal. A cistoscopia revelou a existência de pequenos tumores na bexiga, e a ressecção transuretral foi realizada com o resultado anatomopatológico do carcinoma urotelial superficial da bexiga. Referido por: Carlos Melo - Avenida Dos Aliados, 56, 22 Espanha E-mail: maria.prado@jacob.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|pt| |Size:|1.3 GB| ## Included Models - nlp.DocumentAssembler - nlp.SentenceDetectorDLModel - nlp.TokenizerModel - nlp.WordEmbeddingsModel - medical.NerModel - nlp.NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - nlp.RegexMatcherModel - nlp.RegexMatcherModel - ChunkMergeModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - medical.DeIdentificationModel - Finisher --- layout: model title: English asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11 TFWav2Vec2ForCTC from nimrah author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11` is a English model originally trained by nimrah. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11_en_4.2.0_3.0_1664117519491.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11_en_4.2.0_3.0_1664117519491.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_hindi_home_colab_11| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191119508.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657191119508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_hier_triplet_0.1_epochs_1_shard_1_squad2.0 --- layout: model title: Korean BertForQuestionAnswering Base Cased model (from tucan9389) author: John Snow Labs name: bert_qa_kcbert_base_finetuned_squad date: 2022-07-07 tags: [ko, open_source, bert, question_answering] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `kcbert-base-finetuned-squad` is a Korean model originally trained by `tucan9389`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_kcbert_base_finetuned_squad_ko_4.0.0_3.0_1657189537948.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_kcbert_base_finetuned_squad_ko_4.0.0_3.0_1657189537948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_kcbert_base_finetuned_squad","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_kcbert_base_finetuned_squad","ko") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_kcbert_base_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|406.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tucan9389/kcbert-base-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Tongan author: John Snow Labs name: opus_mt_en_to date: 2020-12-29 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, to, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `to` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_to_xx_2.7.0_2.4_1609254398876.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_to_xx_2.7.0_2.4_1609254398876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_to| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish BertForQuestionAnswering model (from MMG) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG date: 2022-06-03 tags: [es, open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-squad2-es` is a Spanish model orginally trained by `MMG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG_es_4.0.0_3.0_1654249761122.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG_es_4.0.0_3.0_1654249761122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2.bert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_squad2_es_MMG| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MMG/bert-base-spanish-wwm-cased-finetuned-squad2-es --- layout: model title: French T5ForConditionalGeneration Cased model (from ZakaryaRouzki) author: John Snow Labs name: t5_punctuation date: 2023-01-31 tags: [fr, open_source, t5, tensorflow] task: Text Generation language: fr edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-punctuation` is a French model originally trained by `ZakaryaRouzki`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_punctuation_fr_4.3.0_3.0_1675125343642.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_punctuation_fr_4.3.0_3.0_1675125343642.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_punctuation","fr") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_punctuation","fr") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_punctuation| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|fr| |Size:|925.1 MB| ## References - https://huggingface.co/ZakaryaRouzki/t5-punctuation - https://linkedin.com/in/rouzki --- layout: model title: Stopwords Remover for Telugu language (52 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, te, open_source] task: Stop Words Removal language: te edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_te_3.4.1_3.0_1646672989663.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_te_3.4.1_3.0_1646672989663.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+ |result | +------------------------------+ |[మీరు, నన్ను, కంటే, మెరుగైనది]| +------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|te| |Size:|1.7 KB| --- layout: model title: Spanish NER for Laws and Treaties/Agreements (Roberta) author: John Snow Labs name: legner_laws_treaties date: 2022-09-28 tags: [es, legal, ner, laws, treaties, agreements, licensed] task: Named Entity Recognition language: es edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Legal Roberta Named Entity Recognition model in Spanish, able to recognize the following entities: - LEY: Law - TRAT_INTL: International Treaty (Agreement) This model originally trained on scjn dataset, available [here](https://huggingface.co/datasets/scjnugacj/scjn_dataset_ner) and finetuned on scrapped documents (as, for example, [this one](https://www.wipo.int/export/sites/www/pct/es/texts/pdf/pct.pdf)), improving the coverage of the original version, published [here](https://huggingface.co/datasets/scjnugacj/scjn_dataset_ner). ## Predicted Entities `LAW`, `TRAT_INTL` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_laws_treaties_es_1.0.0_3.0_1664362398391.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_laws_treaties_es_1.0.0_3.0_1664362398391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash para la Protección de la Propiedad Industrial. (TRAT_INTL) ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_laws_treaties| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model was originally trained on scjn dataset, available [here](https://huggingface.co/datasets/scjnugacj/scjn_dataset_ner) and finetuned on scrapped documents (as, for example, [this one](https://www.wipo.int/export/sites/www/pct/es/texts/pdf/pct.pdf)), improving the coverage of the original version, published [here](https://huggingface.co/datasets/scjnugacj/scjn_dataset_ner). ## Benchmarking ```bash label prec rec f1 Macro-average 0.9361195 0.9294152 0.9368145 Micro-average 0.9856711 0.9857456 0.9851656 ``` --- layout: model title: Detect Family History Status from Oncology Entities author: John Snow Labs name: assertion_oncology_family_history_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion, family_history] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects entities refering to the family history. ## Predicted Entities `Family_History`, `Other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_family_history_wip_en_4.1.0_3.0_1664642142116.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_family_history_wip_en_4.1.0_3.0_1664642142116.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------------|:------------|:---------------| | breast cancer | Cancer_Dx | Family_History | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_family_history_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Family_History 0.94 0.84 0.89 37.0 Other 0.91 0.97 0.94 62.0 macro-avg 0.92 0.90 0.91 99.0 weighted-avg 0.92 0.92 0.92 99.0 ``` --- layout: model title: RCT Binary Classifier (BioBERT) Pipeline author: John Snow Labs name: bert_sequence_classifier_binary_rct_biobert_pipeline date: 2023-06-13 tags: [licensed, classifier, rct, clinical, en] task: Entity Resolution language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pre-trained pipeline is a BioBERT based classifier that can classify if an article is a randomized clinical trial (RCT) or not. This pretrained pipeline is built on the top of [bert_sequence_classifier_binary_rct_biobert](https://nlp.johnsnowlabs.com/2022/04/25/bert_sequence_classifier_binary_rct_biobert_en_3_0.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_binary_rct_biobert_pipeline_en_4.4.4_3.2_1686665514038.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_binary_rct_biobert_pipeline_en_4.4.4_3.2_1686665514038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_sequence_classifier_binary_rct_biobert_pipeline", "en", "clinical/models") result = pipeline.annotate("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_sequence_classifier_binary_rct_biobert_pipeline", "en", "clinical/models") val result = pipeline.annotate("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_sequence.binary_rct_biobert.pipeline").predict("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ```

## Results ```bash Results +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |rct |text | +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |True|Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. | +----+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_binary_rct_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|406.0 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForSequenceClassification --- layout: model title: Turkish BertForQuestionAnswering Base Cased model (from husnu) author: John Snow Labs name: bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1 date: 2022-07-07 tags: [tr, open_source, bert, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-turkish-128k-cased-finetuned_lr-2e-05_epochs-3TQUAD2-finetuned_lr-2e-05_epochs-1` is a Turkish model originally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1_tr_4.0.0_3.0_1657183615493.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1_tr_4.0.0_3.0_1657183615493.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_turkish_128k_cased_tquad2_finetuned_lr_2e_05_epochs_1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|689.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/husnu/bert-base-turkish-128k-cased-finetuned_lr-2e-05_epochs-3TQUAD2-finetuned_lr-2e-05_epochs-1 --- layout: model title: Legal Transport Policy Document Classifier (EURLEX) author: John Snow Labs name: legclf_transport_policy_bert date: 2023-03-06 tags: [en, legal, classification, clauses, transport_policy, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_transport_policy_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Transport_Policy or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Transport_Policy`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_transport_policy_bert_en_1.0.0_3.0_1678111909241.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_transport_policy_bert_en_1.0.0_3.0_1678111909241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_transport_policy_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Transport_Policy]| |[Other]| |[Other]| |[Transport_Policy]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_transport_policy_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.3 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.76 0.97 0.86 40 Transport_Policy 0.98 0.78 0.87 55 accuracy - - 0.86 95 macro-avg 0.87 0.88 0.86 95 weighted-avg 0.89 0.86 0.86 95 ``` --- layout: model title: Depression Binary Classifier (PHS-BERT) author: John Snow Labs name: bert_sequence_classifier_depression_binary date: 2022-08-10 tags: [public_health, en, licensed, sequence_classification, mental_health, depression] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [PHS-BERT](https://arxiv.org/abs/2204.04521) based text classification model that can classify whether a social media text expresses depression or not. ## Predicted Entities `no-depression`, `depression` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_depression_binary_en_4.0.2_3.0_1660145810767.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_depression_binary_en_4.0.2_3.0_1660145810767.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vinitharaj_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vinitharaj_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vinitharaj_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vinitharaj/distilbert-base-uncased-finetuned-squad --- layout: model title: Multilingual (Finnish, Estonian, English) Bert Embeddings (Base) author: John Snow Labs name: bert_embeddings_finest_bert date: 2022-04-11 tags: [bert, embeddings, fi, et, en, xx, multilingual, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `finest-bert` is a English model orginally trained by `EMBEDDIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_finest_bert_en_3.4.2_3.0_1649672041625.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_finest_bert_en_3.4.2_3.0_1649672041625.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_finest_bert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|538.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/EMBEDDIA/finest-bert - https://arxiv.org/abs/2006.07890 --- layout: model title: Deidentify (Enriched) author: John Snow Labs name: deidentify_enriched_clinical date: 2021-01-29 task: De-identification language: en nav_key: models edition: Spark NLP for Healthcare 2.7.2 spark_version: 2.4 tags: [deidentify, en, obfuscation, licensed] supported: true annotator: DeIdentificationModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Deidentify (Large) is a deidentification model. It identifies instances of protected health information in text documents, and it can either obfuscate them (e.g., replacing names with different, fake names) or mask them (e.g., replacing “2020-06-04” with Some faker data). This model is useful for maintaining HIPAA compliance when dealing with text documents that contain protected health information. ## Predicted Entities - PHONE - PATIENT - COUNTRY - USERNAME - LOCATION-OTHER - DATE - ID - DOCTOR - HOSPITAL - IDNUM - AGE - MEDICALRECORD - CITY - FAX - ZIP - HEALTHPLAN - PROFESSION - BIOID - URL - EMAIL - STATE - ORGANIZATION - STREET - DEVICE {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/deidentify_enriched_clinical_en_2.7.2_2.4_1611917177874.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/deidentify_enriched_clinical_en_2.7.2_2.4_1611917177874.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash sentence deidentified 0 A . A . 1 Record date : 2093-01-13 , David Hale , M.D . Record date : 2093-01-18 , DR. Gregory Kaiser , M.D . 2 , Name : Hendrickson , Ora MR . , Name : Joel Vasquez MR . 3 # 7194334 Date : 01/13/93 PCP : Oliveira , 25 years-old , Record date : 2079-11-09 . # 67696 Date : 01/18/93 PCP : DR. Jennifer Eaton , 25 years-old , Record date : 2079-11-14 . 4 Cocke County Baptist Hospital . San Leandro Hospital – San Leandro . 5 0295 Keats Street 3744 Retreat Avenue ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deidentify_enriched_clinical| |Compatibility:|Spark NLP 2.7.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, ner_chunk]| |Output Labels:|[deidentified]| |Language:|en| --- layout: model title: Fast Neural Machine Translation Model from Arabic to German author: John Snow Labs name: opus_mt_ar_de date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, de, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: de {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_de_xx_3.1.0_2.4_1622562969038.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_de_xx_3.1.0_2.4_1622562969038.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_de| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: RxNorm to MeSH Code Mapping author: John Snow Labs name: rxnorm_mesh_mapping date: 2021-07-01 tags: [rxnorm, mesh, en, licensed, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps RxNorm codes to MeSH codes without using any text data. You’ll just feed white space-delimited RxNorm codes and it will return the corresponding MeSH codes as a list. If there is no mapping, the original code is returned with no mapping. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_CODE_MAPPING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_mesh_mapping_en_3.1.0_2.4_1625126293744.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_mesh_mapping_en_3.1.0_2.4_1625126293744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash {'rxnorm': ['1191', '6809', '47613'], 'mesh': ['D001241', 'D008687', 'D019355']} Note: | RxNorm | Details | | ---------- | -------------------:| | 1191 | aspirin | | 6809 | metformin | | 47613 | calcium citrate | | MeSH | Details | | ---------- | -------------------:| | D001241 | Aspirin | | D008687 | Metformin | | D019355 | Calcium Citrate | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_mesh_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: Translate Luba-Lulua to English Pipeline author: John Snow Labs name: translate_lua_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, lua, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `lua` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_lua_en_xx_2.7.0_2.4_1609698600492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_lua_en_xx_2.7.0_2.4_1609698600492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_lua_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_lua_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.lua.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_lua_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Igbo Pipeline author: John Snow Labs name: translate_en_ig date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ig, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ig` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ig_xx_2.7.0_2.4_1609691476741.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ig_xx_2.7.0_2.4_1609691476741.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ig", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ig", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ig').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ig| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual XLMRobertaForTokenClassification Cased model (from SkyR) author: John Snow Labs name: xlmroberta_ner_skyr_wikineural_multilingual date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `wikineural-multilingual-ner` is a Multilingual model originally trained by `SkyR`. ## Predicted Entities `ORG`, `LOC`, `PER`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_skyr_wikineural_multilingual_xx_4.1.0_3.0_1660425623483.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_skyr_wikineural_multilingual_xx_4.1.0_3.0_1660425623483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_skyr_wikineural_multilingual","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_skyr_wikineural_multilingual","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_skyr_wikineural_multilingual| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|890.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/SkyR/wikineural-multilingual-ner --- layout: model title: General model for table detection author: John Snow Labs name: ocr_table_detection_general_model date: 2021-09-04 tags: [en, licensed] task: OCR Table Detection & Recognition language: en nav_key: models edition: Visual NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description General model for table detection inspired by https://arxiv.org/abs/2004.12629 ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_table_detection_general_model_en_3.0.0_3.0_1630757579641.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_table_detection_general_model_en_3.0.0_3.0_1630757579641.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use This modes is used by ImageTableDetector

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() binary_to_image.setImageType(ImageType.TYPE_3BYTE_BGR) table_detector = ImageTableDetector .pretrained("general_model_table_detection_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("table_regions") pipeline = PipelineModel(stages=[ binary_to_image, table_detector ]) ``` ```scala var imgDf = spark.read.format("binaryFile").load(imagePath) var bin2imTransformer = new BinaryToImage() bin2imTransformer.setImageType(ImageType.TYPE_3BYTE_BGR) val dataFrame = bin2imTransformer.transform(imgDf) val tableDetector = ImageTableDetector .pretrained("general_model_table_detection_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("table regions") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ocr_table_detection_general_model| |Type:|ocr| |Compatibility:|Visual NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Output Labels:|[table regions]| |Language:|en| --- layout: model title: English ElectraForQuestionAnswering model (from ptran74) Version-4 author: John Snow Labs name: electra_qa_DSPFirst_Finetuning_4 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `DSPFirst-Finetuning-4` is a English model originally trained by `ptran74`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_4_en_4.0.0_3.0_1655919679099.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_DSPFirst_Finetuning_4_en_4.0.0_3.0_1655919679099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_DSPFirst_Finetuning_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.electra.finetuning_4").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_DSPFirst_Finetuning_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ptran74/DSPFirst-Finetuning-4 - https://github.gatech.edu/pages/VIP-ITS/textbook_SQuAD_explore/explore/textbookv1.0/textbook/ - https://github.com/patil-suraj/question_generation - https://colab.research.google.com/drive/1dJXNstk2NSenwzdtl9xA8AqjP4LL-Ks_?usp=sharing --- layout: model title: English Bert Embeddings Cased model (from antoinev17) author: John Snow Labs name: bert_embeddings_base_uncased_issues_128 date: 2023-02-20 tags: [open_source, bert, bert_embeddings, bertformaskedlm, en, tensorflow] task: Embeddings language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertEmbeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-issues-128` is a English model originally trained by `antoinev17 `. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uncased_issues_128_en_4.3.0_3.0_1676927301180.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_uncased_issues_128_en_4.3.0_3.0_1676927301180.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_base_uncased_issues_128","ko") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_base_uncased_issues_128","ko") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_uncased_issues_128| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|410.1 MB| |Case sensitive:|true| ## References https://huggingface.co/antoinev17/bert-base-uncased-issues-128 --- layout: model title: English BertForQuestionAnswering Cased model (from wiselinjayajos) author: John Snow Labs name: bert_qa_wiselinjayajos_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `wiselinjayajos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_wiselinjayajos_finetuned_squad_en_4.0.0_3.0_1657186807960.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_wiselinjayajos_finetuned_squad_en_4.0.0_3.0_1657186807960.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_wiselinjayajos_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_wiselinjayajos_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_wiselinjayajos_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/wiselinjayajos/bert-finetuned-squad --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_becasincentivos6 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-becasIncentivos6` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos6_es_4.3.0_3.0_1674218206378.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos6_es_4.3.0_3.0_1674218206378.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos6","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos6","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_becasincentivos6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|459.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-becasIncentivos6 --- layout: model title: Legal Method of payment Clause Binary Classifier (md) author: John Snow Labs name: legclf_method_of_payment_md date: 2022-11-25 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `method-of-payment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `method-of-payment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_method_of_payment_md_en_1.0.0_3.0_1669376507093.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_method_of_payment_md_en_1.0.0_3.0_1669376507093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = legal.ClassifierDLModel.pretrained("legclf_method_of_payment_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[method-of-payment]| |[other]| |[other]| |[method-of-payment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_method_of_payment_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support method-of-payment 0.97 0.94 0.96 35 other 0.95 0.97 0.96 39 accuracy 0.96 74 macro avg 0.96 0.96 0.96 74 weighted avg 0.96 0.96 0.96 74 ``` --- layout: model title: English T5ForConditionalGeneration Mini Cased model (from google) author: John Snow Labs name: t5_efficient_mini_nl12 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-mini-nl12` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_mini_nl12_en_4.3.0_3.0_1675118188776.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_mini_nl12_en_4.3.0_3.0_1675118188776.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_mini_nl12","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_mini_nl12","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_mini_nl12| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|156.4 MB| ## References - https://huggingface.co/google/t5-efficient-mini-nl12 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal NER for NDA (Preamble Clause) author: John Snow Labs name: legner_nda_preamble date: 2023-04-06 tags: [en, licensed, legal, ner, nda, preamble, purpose] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `PREAMBLE` clause with a proper classifier (use legmulticlf_mnda_sections_paragraph_other for that purpose). It will extract the following entities: `PURPOSE`, and `PURPOSE_OBJECT`. ## Predicted Entities `PURPOSE`, `PURPOSE_OBJECT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_preamble_en_1.0.0_3.0_1680791988031.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_preamble_en_1.0.0_3.0_1680791988031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_preamble", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""In order to facilitate the consideration and negotiation of a possible transaction involving Chordiant and Pegasystems ( referred to collectively as the "Parties" and individually as a "Party"), each Party has requested access to certain non-public information regarding the other Party and the other Party’s subsidiaries."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-------------+--------------+ |chunk |ner_label | +-------------+--------------+ |consideration|PURPOSE | |negotiation |PURPOSE | |transaction |PURPOSE_OBJECT| +-------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_preamble| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support B-PURPOSE 1.00 0.93 0.97 15 B-PURPOSE_OBJECT 0.90 0.82 0.86 11 I-PURPOSE_OBJECT 1.00 0.80 0.89 5 micro-avg 0.96 0.87 0.92 31 macro-avg 0.97 0.85 0.90 31 weighted-avg 0.96 0.87 0.91 31 ``` --- layout: model title: Sentence Entity Resolver for ICD-O (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_icdo date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD-O codes (Topography & Morphology codes) using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Given an oncological entity found in the text (via NER models like ner_jsl), it returns top terms and resolutions along with the corresponding ICD-O codes to present more granularity with respect to body parts mentioned. It also returns the original `Topography` codes, `Morphology` codes comprising of `Histology` and `Behavior` codes, and descriptions in the aux metadata. ## Predicted Entities Predicts ICD-O Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icdo_en_3.0.4_3.0_1621191532225.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icdo_en_3.0.4_3.0_1621191532225.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------+-----+---+---------+------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM|8312/3| 0.3558|Renal cell carcin...|8312/3:::9964/3::...| |chronic renal ins...| 83|109| PROBLEM|9980/3| 0.5290|Refractory anemia...|9980/3:::8312/3::...| | COPD| 113|116| PROBLEM|9950/3| 0.2092|Polycythemia vera...|9950/3:::8141/3::...| | gastritis| 120|128| PROBLEM|8150/3| 0.1795|Islet cell carcin...|8150/3:::8153/3::...| | TIA| 136|138| PROBLEM|9570/0| 0.4843|Neuroma, NOS:::Ca...|9570/0:::8692/3::...| |a non-ST elevatio...| 182|202| PROBLEM|8343/2| 0.1914|Non-invasive EFVP...|8343/2:::9150/0::...| |Guaiac positive s...| 208|229| PROBLEM|8155/3| 0.1069|Vipoma:::Myeloid ...|8155/3:::9930/3::...| |cardiac catheteri...| 295|317| TEST|8045/3| 0.1144|Combined small ce...|8045/3:::9705/3::...| | PTCA| 324|327|TREATMENT|9735/3| 0.0924|Plasmablastic lym...|9735/3:::9365/3::...| | mid LAD lesion| 332|345| PROBLEM|9383/1| 0.0845|Subependymoma:::D...|9383/1:::8806/3::...| +--------------------+-----+---+---------+------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icdo| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[icdo_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on ICD-O Histology Behaviour dataset with ``sbiobert_base_cased_mli`` sentence embeddings. https://apps.who.int/iris/bitstream/handle/10665/96612/9789241548496_eng.pdf --- layout: model title: Multilingual BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_es_hi_dev date: 2022-07-07 tags: [xx, open_source, bert, question_answering] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-es-hi` is a Multilingual model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_es_hi_dev_xx_4.0.0_3.0_1657190135587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_es_hi_dev_xx_4.0.0_3.0_1657190135587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_es_hi_dev","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_es_hi_dev","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_es_hi_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-es-hi --- layout: model title: English BertForQuestionAnswering model (from z-uo) author: John Snow Labs name: bert_qa_bert_qasper date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-qasper` is a English model orginally trained by `z-uo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_qasper_en_4.0.0_3.0_1654184675001.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_qasper_en_4.0.0_3.0_1654184675001.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_qasper","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_qasper","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_z-uo").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_qasper| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/z-uo/bert-qasper --- layout: model title: Classifier for Genders - SBERT author: John Snow Labs name: classifierdl_gender_sbert date: 2020-12-16 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.6.5 spark_version: 2.4 tags: [classifier, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model classifies the gender of the patient in the clinical document. {:.h2_title} ## Predicted Entities `Female`, `Male`, `Unknown`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CLINICAL_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_sbert_en_2.6.4_2.4_1608119379496.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_sbert_en_2.6.4_2.4_1608119379496.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use To classify your text, you can use this model as part of an nlp pipeline with the following stages: DocumentAssembler, BertSentenceEmbeddings (``sbiobert_base_cased_mli``), ClassifierDLModel.

{:.h2_title} ## Results ```bash Female ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_gender_sbert| |Type:|ClassifierDLModel| |Compatibility:|Healthcare NLP 2.6.5 +| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|[en]| |Case sensitive:|True| {:.h2_title} ## Data Source This model is trained on more than four thousands clinical documents (radiology reports, pathology reports, clinical visits etc.), annotated internally. {:.h2_title} ## Benchmarking ```bash label precision recall f1-score support Female 0.9224 0.8954 0.9087 239 Male 0.7895 0.8468 0.8171 124 Unknown 0.8077 0.7778 0.7925 54 accuracy - - 0.8657 417 macro-avg 0.8399 0.8400 0.8394 417 weighted-avg 0.8680 0.8657 0.8664 417 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from seomh) author: John Snow Labs name: distilbert_qa_seomh_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `seomh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_seomh_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772473886.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_seomh_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772473886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_seomh_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_seomh_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_seomh_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/seomh/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec_large_xlsr_korean TFWav2Vec2ForCTC from fleek author: John Snow Labs name: asr_wav2vec_large_xlsr_korean date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec_large_xlsr_korean` is a English model originally trained by fleek. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec_large_xlsr_korean_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec_large_xlsr_korean_en_4.2.0_3.0_1664098551065.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec_large_xlsr_korean_en_4.2.0_3.0_1664098551065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec_large_xlsr_korean", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec_large_xlsr_korean", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec_large_xlsr_korean| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Legal Assignment And Subletting Clause Binary Classifier author: John Snow Labs name: legclf_assignment_and_subletting_clause date: 2023-01-29 tags: [en, legal, classification, assignment, subletting, clauses, assignment_and_subletting, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `assignment-and-subletting` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `assignment-and-subletting`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_assignment_and_subletting_clause_en_1.0.0_3.0_1674993574865.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_assignment_and_subletting_clause_en_1.0.0_3.0_1674993574865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_assignment_and_subletting_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[assignment-and-subletting]| |[other]| |[other]| |[assignment-and-subletting]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_assignment_and_subletting_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support assignment-and-subletting 1.00 0.96 0.98 26 other 0.97 1.00 0.99 38 accuracy - - 0.98 64 macro-avg 0.99 0.98 0.98 64 weighted-avg 0.98 0.98 0.98 64 ``` --- layout: model title: Financial Equity Item Binary Classifier author: John Snow Labs name: finclf_equity_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `equity` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `equity` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_equity_item_en_1.0.0_3.2_1660154390458.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_equity_item_en_1.0.0_3.2_1660154390458.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_equity_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[equity]| |[other]| |[other]| |[equity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_equity_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support equity 0.84 0.85 0.84 66 other 0.85 0.84 0.84 67 accuracy - - 0.84 133 macro-avg 0.84 0.84 0.84 133 weighted-avg 0.84 0.84 0.84 133 ``` --- layout: model title: Legal No Defaults Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_no_defaults_bert date: 2023-03-05 tags: [en, legal, classification, clauses, no_defaults, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `No_Defaults` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `No_Defaults`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_defaults_bert_en_1.0.0_3.0_1678049951854.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_defaults_bert_en_1.0.0_3.0_1678049951854.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_no_defaults_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[No_Defaults]| |[Other]| |[Other]| |[No_Defaults]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_defaults_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support No_Defaults 0.90 0.95 0.92 37 Other 0.96 0.93 0.95 56 accuracy - - 0.94 93 macro-avg 0.93 0.94 0.93 93 weighted-avg 0.94 0.94 0.94 93 ``` --- layout: model title: German RobertaForMaskedLM Cased model (from FabianGroeger) author: John Snow Labs name: roberta_embeddings_hotelbert date: 2022-12-12 tags: [de, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `HotelBERT` is a German model originally trained by `FabianGroeger`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_hotelbert_de_4.2.4_3.0_1670858469747.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_hotelbert_de_4.2.4_3.0_1670858469747.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_hotelbert","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_hotelbert","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_hotelbert| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|411.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/FabianGroeger/HotelBERT --- layout: model title: English asr_Urdu_repo TFWav2Vec2ForCTC from bilalahmed15 author: John Snow Labs name: pipeline_asr_Urdu_repo date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Urdu_repo` is a English model originally trained by bilalahmed15. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Urdu_repo_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Urdu_repo_en_4.2.0_3.0_1664107452190.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Urdu_repo_en_4.2.0_3.0_1664107452190.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Urdu_repo', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Urdu_repo", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Urdu_repo| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Guarantee Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_guarantee_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, guarantee, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_guarantee_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `guarantee-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `guarantee-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_guarantee_agreement_bert_en_1.0.0_3.0_1669313872907.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_guarantee_agreement_bert_en_1.0.0_3.0_1669313872907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_guarantee_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[guarantee-agreement]| |[other]| |[other]| |[guarantee-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_guarantee_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support guarantee-agreement 0.88 0.88 0.88 33 other 0.94 0.94 0.94 65 accuracy - - 0.92 98 macro-avg 0.91 0.91 0.91 98 weighted-avg 0.92 0.92 0.92 98 ``` --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Polish (WikiNER 6B 300) author: John Snow Labs name: wikiner_6B_300 date: 2020-05-10 task: Named Entity Recognition language: pl edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [ner, pl, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 6B 300 is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_PL){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_PL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_pl_2.5.0_2.4_1588519719571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_pl_2.5.0_2.4_1588519719571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PER | |To amerykański |MISC | |Najbardziej |MISC | |Microsoft Corporation |ORG | |Podczas swojej |MISC | |Microsoft Gates |MISC | |CEO |ORG | |Urodzony |ORG | |Seattle |LOC | |Waszyngton |LOC | |Gates |PER | |Microsoftu |ORG | |Paulem Allenem |PER | |W Albuquerque |MISC | |Nowym Meksyku |LOC | |Gates |PER | |Ale |PER | |Gates |PER | |Opinię |MISC | |W czerwcu 2006 |MISC | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pl| |Case sensitive:|false| {:.h2_title} ## Data Source The model was trained based on data from [https://pl.wikipedia.org](https://pl.wikipedia.org) --- layout: model title: English image_classifier_vit_autotrain_fashion_mnist__base ViTForImageClassification from abhishek author: John Snow Labs name: image_classifier_vit_autotrain_fashion_mnist__base date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_autotrain_fashion_mnist__base` is a English model originally trained by abhishek. ## Predicted Entities `Coat`, `Shirt`, `Sneaker`, `Sandal`, `T - shirt / top`, `Bag`, `Trouser`, `Dress`, `Ankle boot`, `Pullover` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_autotrain_fashion_mnist__base_en_4.1.0_3.0_1660170414663.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_autotrain_fashion_mnist__base_en_4.1.0_3.0_1660170414663.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_autotrain_fashion_mnist__base", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_autotrain_fashion_mnist__base", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_autotrain_fashion_mnist__base| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Sentiment Analysis Pipeline for Turkish texts author: John Snow Labs name: classifierdl_use_sentiment_pipeline date: 2021-11-04 tags: [turkish, sentiment, tr, open_source] task: Sentiment Analysis language: tr edition: Spark NLP 3.3.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline identifies the sentiments (positive or negative) in Turkish texts. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_sentiment_pipeline_tr_3.3.1_2.4_1636020950989.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_sentiment_pipeline_tr_3.3.1_2.4_1636020950989.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_use_sentiment_pipeline", lang = "tr") result1 = pipeline.annotate("Bu sıralar kafam çok karışık.") result2 = pipeline.annotate("Sınavımı geçtiğimi öğrenince derin bir nefes aldım.") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_bert_sentiment_pipeline", lang = "de") val result1 = pipeline.fullAnnotate("Bu sıralar kafam çok karışık.")(0) val result2 = pipeline.fullAnnotate("Sınavımı geçtiğimi öğrenince derin bir nefes aldım.")(0) ```

## Results ```bash ['NEGATIVE'] ['POSITIVE'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_sentiment_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.1+| |License:|Open Source| |Edition:|Official| |Language:|tr| ## Included Models - DocumentAssembler - UniversalSentenceEncoder - ClassifierDLModel --- layout: model title: Fast Neural Machine Translation Model from Czech to English author: John Snow Labs name: opus_mt_cs_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, cs, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `cs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_cs_en_xx_2.7.0_2.4_1609166719928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_cs_en_xx_2.7.0_2.4_1609166719928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_cs_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Clinical Entities author: John Snow Labs name: ner_jsl_limited_80p_for_benchmarks date: 2023-04-02 tags: [ner, licensed, en, clinical, benchmark] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description `Important Note:` This model is trained with a partial dataset that is used to train [ner_jsl](https://nlp.johnsnowlabs.com/2022/10/19/ner_jsl_en.html); and meant to be used for benchmarking run at [LLMs Healthcare Benchmarks](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/academic/LLMs_in_Healthcare). Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state-of-the-art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. This model is the official version of jsl_ner_wip_clinical model. Definitions of Predicted Entities: - `Injury_or_Poisoning`: Physical harm or injury caused to the body, including those caused by accidents, falls, or poisoning of a patient or someone else. - `Direction`: All the information relating to the laterality of the internal and external organs. - `Test`: Mentions of laboratory, pathology, and radiological tests. - `Admission_Discharge`: Terms that indicate the admission and/or the discharge of a patient. - `Death_Entity`: Mentions that indicate the death of a patient. - `Relationship_Status`: State of patients romantic or social relationships (e.g. single, married, divorced). - `Duration`: The duration of a medical treatment or medication use. - `Respiration`: Number of breaths per minute. - `Hyperlipidemia`: Terms that indicate hyperlipidemia with relevant subtypes and synonims. - `Birth_Entity`: Mentions that indicate giving birth. - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Labour_Delivery`: Extractions include stages of labor and delivery. - `Family_History_Header`: identifies section headers that correspond to Family History of the patient. - `BMI`: Numeric values and other text information related to Body Mass Index. - `Temperature`: All mentions that refer to body temperature. - `Alcohol`: Terms that indicate alcohol use, abuse or drinking issues of a patient or someone else. - `Kidney_Disease`: Terms that refer to any kidney diseases (includes mentions of modifiers such as "Acute" or "Chronic"). - `Oncological`: All the cancer, tumor or metastasis related extractions mentioned in the document, of the patient or someone else. - `Medical_History_Header`: Identifies section headers that correspond to Past Medical History of a patient. - `Cerebrovascular_Disease`: All terms that refer to cerebrovascular diseases and events. - `Oxygen_Therapy`: Breathing support triggered by patient or entirely or partially by machine (e.g. ventilator, BPAP, CPAP). - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Psychological_Condition`: All the Mental health diagnosis, disorders, conditions or syndromes of a patient or someone else. - `Heart_Disease`: All mentions of acquired, congenital or degenerative heart diseases. - `Employment`: All mentions of patient or provider occupational titles and employment status . - `Obesity`: Terms related to a patient being obese (overweight and BMI are extracted as different labels). - `Disease_Syndrome_Disorder`: All the diseases mentioned in the document, of the patient or someone else (excluding diseases that are extracted with their specific labels, such as "Heart_Disease" etc.). - `Pregnancy`: All terms related to Pregnancy (excluding terms that are extracted with their specific labels, such as "Labour_Delivery" etc.). - `ImagingFindings`: All mentions of radiographic and imagistic findings. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Medical_Device`: All mentions related to medical devices and supplies. - `Race_Ethnicity`: All terms that refer to racial and national origin of sociocultural groups. - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Treatment`: Includes therapeutic and minimally invasive treatment and procedures (invasive treatments or procedures are extracted as "Procedure"). - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_Ingredient`: Active ingredient/s found in drug products. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Diet`: All mentions and information regarding patients dietary habits. - `External_body_part_or_region`: All mentions related to external body parts or organs that can be examined by naked eye. - `LDL`: All mentions related to the lab test and results for LDL (Low Density Lipoprotein). - `VS_Finding`: Qualitative data (e.g. Fever, Cyanosis, Tachycardia) and any other symptoms that refers to vital signs. - `Allergen`: Allergen related extractions mentioned in the document. - `EKG_Findings`: All mentions of EKG readings. - `Imaging_Technique`: All mentions of special radiographic views or special imaging techniques used in radiology. - `Triglycerides`: All mentions terms related to specific lab test for Triglycerides. - `RelativeTime`: Time references that are relative to different times or events (e.g. words such as "approximately", "in the morning"). - `Gender`: Gender-specific nouns and pronouns. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Social_History_Header`: Identifies section headers that correspond to Social History of a patient. - `Substance_Quantity`: All mentions of substance quantity (quantitative information related to illicit/recreational drugs). - `Diabetes`: All terms related to diabetes mellitus. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `Internal_organ_or_component`: All mentions related to internal body parts or organs that can not be examined by naked eye. - `Clinical_Dept`: Terms that indicate the medical and/or surgical departments. - `Form`: Drug and medication forms available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_BrandName`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Strength`: Potency of one unit of drug (or a combination of drugs) the measurement units available are described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Fetus_NewBorn`: All terms related to fetus, infant, new born (excluding terms that are extracted with their specific labels, such as "Labour_Delivery", "Pregnancy" etc.). - `RelativeDate`: Temporal references that are relative to the date of the text or to any other specific date (e.g. "approximately two years ago", "about two days ago"). - `Height`: All mentions related to a patients height. - `Test_Result`: Terms related to all the test results present in the document (clinical tests results are included). - `Sexually_Active_or_Sexual_Orientation`: All terms that are related to sexuality, sexual orientations and sexual activity. - `Frequency`: Frequency of administration for a dose prescribed. - `Time`: Specific time references (hour and/or minutes). - `Weight`: All mentions related to a patients weight. - `Vaccine`: Generic and brand name of vaccines or vaccination procedure. - `Vital_Signs_Header`: Identifies section headers that correspond to Vital Signs of a patient. - `Communicable_Disease`: Includes all mentions of communicable diseases. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Overweight`: Terms related to the patient being overweight (BMI and Obesity is extracted separately). - `Hypertension`: All terms related to Hypertension (quantitative data such as 150/100 is extracted as Blood_Pressure). - `HDL`: Terms related to the lab test for HDL (High Density Lipoprotein). - `Total_Cholesterol`: Terms related to the lab test and results for cholesterol. - `Smoking`: All mentions of smoking status of a patient. - `Date`: Mentions of an exact date, in any format, including day number, month and/or year. ## Predicted Entities `Injury_or_Poisoning`, `Direction`, `Test`, `Admission_Discharge`, `Death_Entity`, `Relationship_Status`, `Duration`, `Respiration`, `Hyperlipidemia`, `Birth_Entity`, `Age`, `Labour_Delivery`, `Family_History_Header`, `BMI`, `Temperature`, `Alcohol`, `Kidney_Disease`, `Oncological`, `Medical_History_Header`, `Cerebrovascular_Disease`, `Oxygen_Therapy`, `O2_Saturation`, `Psychological_Condition`, `Heart_Disease`, `Employment`, `Obesity`, `Disease_Syndrome_Disorder`, `Pregnancy`, `ImagingFindings`, `Procedure`, `Medical_Device`, `Race_Ethnicity`, `Section_Header`, `Symptom`, `Treatment`, `Substance`, `Route`, `Drug_Ingredient`, `Blood_Pressure`, `Diet`, `External_body_part_or_region`, `LDL`, `VS_Finding`, `Allergen`, `EKG_Findings`, `Imaging_Technique`, `Triglycerides`, `RelativeTime`, `Gender`, `Pulse`, `Social_History_Header`, `Substance_Quantity`, `Diabetes`, `Modifier`, `Internal_organ_or_component`, `Clinical_Dept`, `Form`, `Drug_BrandName`, `Strength`, `Fetus_NewBorn`, `RelativeDate`, `Height`, `Test_Result`, `Sexually_Active_or_Sexual_Orientation`, `Frequency`, `Time`, `Weight`, `Vaccine`, `Vaccine_Name`, `Vital_Signs_Header`, `Communicable_Disease`, `Dosage`, `Overweight`, `Hypertension`, `HDL`, `Total_Cholesterol`, `Smoking`, `Date` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_limited_80p_for_benchmarks_en_4.3.2_3.0_1680468591578.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_limited_80p_for_benchmarks_en_4.3.2_3.0_1680468591578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | begin | end | sentence_id | entities | |---:|:------------------------------------------|--------:|------:|--------------:|:-----------------------------| | 0 | 21-day-old | 18 | 27 | 0 | Age | | 1 | Caucasian | 29 | 37 | 0 | Race_Ethnicity | | 2 | male | 39 | 42 | 0 | Gender | | 3 | 2 days | 53 | 58 | 0 | Duration | | 4 | congestion | 63 | 72 | 0 | Symptom | | 5 | mom | 76 | 78 | 0 | Gender | | 6 | suctioning yellow discharge | 89 | 115 | 0 | Symptom | | 7 | nares | 136 | 140 | 0 | External_body_part_or_region | | 8 | she | 148 | 150 | 0 | Gender | | 9 | mild | 169 | 172 | 0 | Modifier | | 10 | problems with his breathing while feeding | 174 | 214 | 0 | Symptom | | 11 | perioral cyanosis | 238 | 254 | 0 | Symptom | | 12 | retractions | 259 | 269 | 0 | Symptom | | 13 | Influenza vaccine | 326 | 342 | 1 | Vaccine_Name | | 14 | One day ago | 345 | 355 | 2 | RelativeDate | | 15 | mom | 358 | 360 | 2 | Gender | | 16 | tactile temperature | 377 | 395 | 2 | Symptom | | 17 | Tylenol | 418 | 424 | 2 | Drug_BrandName | | 18 | Baby | 427 | 430 | 3 | Age | | 19 | decreased p.o | 450 | 462 | 3 | Symptom | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_limited_80p_for_benchmarks| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.3 MB| ## References Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash label tp fp fn total precision recall f1 VS_Finding 164.0 69.0 40.0 204.0 0.7039 0.8039 0.7506 Direction 3394.0 452.0 367.0 3761.0 0.8825 0.9024 0.8923 Respiration 74.0 3.0 5.0 79.0 0.961 0.9367 0.9487 Cerebrovascular_D... 103.0 27.0 14.0 117.0 0.7923 0.8803 0.834 Family_History_He... 80.0 1.0 1.0 81.0 0.9877 0.9877 0.9877 Heart_Disease 432.0 74.0 64.0 496.0 0.8538 0.871 0.8623 ImagingFindings 66.0 32.0 102.0 168.0 0.6735 0.3929 0.4962 RelativeTime 103.0 55.0 82.0 185.0 0.6519 0.5568 0.6006 Strength 552.0 56.0 37.0 589.0 0.9079 0.9372 0.9223 Smoking 105.0 3.0 9.0 114.0 0.9722 0.9211 0.9459 Medical_Device 3043.0 530.0 364.0 3407.0 0.8517 0.8932 0.8719 Allergen 1.0 1.0 17.0 18.0 0.5 0.0556 0.1 EKG_Findings 42.0 36.0 53.0 95.0 0.5385 0.4421 0.4855 Pulse 106.0 23.0 17.0 123.0 0.8217 0.8618 0.8413 Psychological_Con... 103.0 35.0 20.0 123.0 0.7464 0.8374 0.7893 Overweight 5.0 3.0 0.0 5.0 0.625 1.0 0.7692 Triglycerides 3.0 0.0 2.0 5.0 1.0 0.6 0.75 Obesity 34.0 4.0 7.0 41.0 0.8947 0.8293 0.8608 Admission_Discharge 283.0 30.0 7.0 290.0 0.9042 0.9759 0.9386 HDL 2.0 1.0 0.0 2.0 0.6667 1.0 0.8 Diabetes 107.0 8.0 3.0 110.0 0.9304 0.9727 0.9511 Section_Header 3184.0 185.0 118.0 3302.0 0.9451 0.9643 0.9546 Age 524.0 49.0 61.0 585.0 0.9145 0.8957 0.905 O2_Saturation 29.0 10.0 13.0 42.0 0.7436 0.6905 0.716 Kidney_Disease 82.0 8.0 17.0 99.0 0.9111 0.8283 0.8677 Test 2063.0 451.0 414.0 2477.0 0.8206 0.8329 0.8267 Communicable_Disease 22.0 9.0 9.0 31.0 0.7097 0.7097 0.7097 Hypertension 124.0 4.0 7.0 131.0 0.9688 0.9466 0.9575 External_body_par... 2277.0 405.0 353.0 2630.0 0.849 0.8658 0.8573 Oxygen_Therapy 70.0 17.0 10.0 80.0 0.8046 0.875 0.8383 Modifier 1960.0 357.0 549.0 2509.0 0.8459 0.7812 0.8123 Test_Result 796.0 178.0 210.0 1006.0 0.8172 0.7913 0.804 BMI 4.0 0.0 1.0 5.0 1.0 0.8 0.8889 Labour_Delivery 55.0 31.0 26.0 81.0 0.6395 0.679 0.6587 Employment 192.0 24.0 48.0 240.0 0.8889 0.8 0.8421 Fetus_NewBorn 24.0 19.0 43.0 67.0 0.5581 0.3582 0.4364 Clinical_Dept 795.0 57.0 85.0 880.0 0.9331 0.9034 0.918 Time 22.0 11.0 9.0 31.0 0.6667 0.7097 0.6875 Procedure 2458.0 413.0 503.0 2961.0 0.8561 0.8301 0.8429 Diet 21.0 6.0 30.0 51.0 0.7778 0.4118 0.5385 Oncological 342.0 62.0 77.0 419.0 0.8465 0.8162 0.8311 LDL 4.0 0.0 0.0 4.0 1.0 1.0 1.0 Symptom 5777.0 1069.0 1277.0 7054.0 0.8439 0.819 0.8312 Temperature 86.0 5.0 12.0 98.0 0.9451 0.8776 0.9101 Vital_Signs_Header 201.0 23.0 14.0 215.0 0.8973 0.9349 0.9157 Relationship_Status 44.0 1.0 3.0 47.0 0.9778 0.9362 0.9565 Total_Cholesterol 10.0 5.0 7.0 17.0 0.6667 0.5882 0.625 Blood_Pressure 131.0 35.0 23.0 154.0 0.7892 0.8506 0.8188 Injury_or_Poisoning 431.0 71.0 140.0 571.0 0.8586 0.7548 0.8034 Drug_Ingredient 1508.0 106.0 158.0 1666.0 0.9343 0.9052 0.9195 Treatment 124.0 36.0 68.0 192.0 0.775 0.6458 0.7045 Pregnancy 89.0 40.0 38.0 127.0 0.6899 0.7008 0.6953 Vaccine 1.0 0.0 4.0 5.0 1.0 0.2 0.3333 Disease_Syndrome_... 2471.0 551.0 432.0 2903.0 0.8177 0.8512 0.8341 Height 12.0 3.0 9.0 21.0 0.8 0.5714 0.6667 Frequency 500.0 103.0 110.0 610.0 0.8292 0.8197 0.8244 Route 797.0 77.0 70.0 867.0 0.9119 0.9193 0.9156 Duration 258.0 52.0 106.0 364.0 0.8323 0.7088 0.7656 Death_Entity 38.0 7.0 3.0 41.0 0.8444 0.9268 0.8837 Internal_organ_or... 5434.0 839.0 984.0 6418.0 0.8663 0.8467 0.8564 Vaccine_Name 5.0 1.0 3.0 8.0 0.8333 0.625 0.7143 Alcohol 78.0 13.0 6.0 84.0 0.8571 0.9286 0.8914 Substance_Quantity 0.0 9.0 1.0 1.0 0.0 0.0 0.0 Date 455.0 25.0 12.0 467.0 0.9479 0.9743 0.9609 Hyperlipidemia 34.0 0.0 2.0 36.0 1.0 0.9444 0.9714 Social_History_He... 75.0 3.0 3.0 78.0 0.9615 0.9615 0.9615 Imaging_Technique 25.0 16.0 23.0 48.0 0.6098 0.5208 0.5618 Race_Ethnicity 110.0 0.0 2.0 112.0 1.0 0.9821 0.991 Drug_BrandName 788.0 65.0 53.0 841.0 0.9238 0.937 0.9303 RelativeDate 488.0 150.0 98.0 586.0 0.7649 0.8328 0.7974 Gender 5189.0 66.0 55.0 5244.0 0.9874 0.9895 0.9885 Dosage 229.0 32.0 69.0 298.0 0.8774 0.7685 0.8193 Form 179.0 22.0 37.0 216.0 0.8905 0.8287 0.8585 Medical_History_H... 112.0 7.0 6.0 118.0 0.9412 0.9492 0.9451 Birth_Entity 2.0 2.0 5.0 7.0 0.5 0.2857 0.3636 Substance 60.0 6.0 11.0 71.0 0.9091 0.8451 0.8759 Sexually_Active_o... 2.0 1.0 1.0 3.0 0.6667 0.6667 0.6667 Weight 77.0 8.0 12.0 89.0 0.9059 0.8652 0.8851 macro - - - - - - 0.7914 micro - - - - - - 0.8691 ``` --- layout: model title: Danish asr_xls_r_300m_nst_cv9 TFWav2Vec2ForCTC from chcaa author: John Snow Labs name: pipeline_asr_xls_r_300m_nst_cv9 date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_300m_nst_cv9` is a Danish model originally trained by chcaa. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xls_r_300m_nst_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_300m_nst_cv9_da_4.2.0_3.0_1664103552039.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_300m_nst_cv9_da_4.2.0_3.0_1664103552039.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_r_300m_nst_cv9', lang = 'da') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_r_300m_nst_cv9", lang = "da") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_r_300m_nst_cv9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|da| |Size:|756.3 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house ViTForImageClassification from mayoughi author: John Snow Labs name: image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house` is a English model originally trained by mayoughi. ## Predicted Entities `coffee house indoors`, `balcony`, `hospital`, `airport`, `hallway` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_en_4.1.0_3.0_1660169311301.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house_en_4.1.0_3.0_1660169311301.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_where_am_I_hospital_balcony_hallway_airport_coffee_house| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_8_h_512 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-8_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_512_zh_4.2.4_3.0_1670021790876.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_8_h_512_zh_4.2.4_3.0_1670021790876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_8_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("jsl_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("jsl_ner_wip_greedy_biobert_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.greedy_wip_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 1 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.9488 | | 2 | male | 38 | 41 | Gender | 0.9978 | | 3 | for 2 days | 48 | 57 | Duration | 0.7709 | | 4 | congestion | 62 | 71 | Symptom | 0.5467 | | 5 | mom | 75 | 77 | Gender | 0.9355 | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | 0.327867 | | 7 | nares | 135 | 139 | External_body_part_or_region | 0.8963 | | 8 | she | 147 | 149 | Gender | 0.995 | | 9 | mild problems with his breathing while feeding | 168 | 213 | Symptom | 0.588714 | | 10 | perioral cyanosis | 237 | 253 | Symptom | 0.58635 | | 11 | retractions | 258 | 268 | Symptom | 0.9864 | | 12 | One day ago | 272 | 282 | RelativeDate | 0.755833 | | 13 | mom | 285 | 287 | Gender | 0.9956 | | 14 | tactile temperature | 304 | 322 | Symptom | 0.10505 | | 15 | Tylenol | 345 | 351 | Drug | 0.9496 | | 16 | Baby | 354 | 357 | Age | 0.976 | | 17 | decreased p.o. intake | 377 | 397 | Symptom | 0.448125 | | 18 | His | 400 | 402 | Gender | 0.999 | | 19 | q.2h. to 5 to 10 minutes | 450 | 473 | Frequency | 0.298843 | | 20 | his | 488 | 490 | Gender | 0.9976 | | 21 | respiratory congestion | 492 | 513 | VS_Finding | 0.6158 | | 22 | He | 516 | 517 | Gender | 0.9998 | | 23 | tired | 550 | 554 | Symptom | 0.8912 | | 24 | fussy | 569 | 573 | Symptom | 0.9541 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_greedy_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: English asr_wav2vec2_japanese_hiragana_vtuber TFWav2Vec2ForCTC from thunninoi author: John Snow Labs name: pipeline_asr_wav2vec2_japanese_hiragana_vtuber date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_japanese_hiragana_vtuber` is a English model originally trained by thunninoi. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_japanese_hiragana_vtuber_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_japanese_hiragana_vtuber_en_4.2.0_3.0_1664095841326.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_japanese_hiragana_vtuber_en_4.2.0_3.0_1664095841326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_japanese_hiragana_vtuber', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_japanese_hiragana_vtuber", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_japanese_hiragana_vtuber| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4_en_4.0.0_3.0_1655732197028.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4_en_4.0.0_3.0_1655732197028.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_256d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_256_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|427.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-4 --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-256-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6_en_4.0.0_3.0_1657192421420.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6_en_4.0.0_3.0_1657192421420.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_256_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|383.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-256-finetuned-squad-seed-6 --- layout: model title: Legal Base salary Clause Binary Classifier author: John Snow Labs name: legclf_base_salary_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `base-salary` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `base-salary` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_base_salary_clause_en_1.0.0_3.2_1660122165052.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_base_salary_clause_en_1.0.0_3.2_1660122165052.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_base_salary_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[base-salary]| |[other]| |[other]| |[base-salary]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_base_salary_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support base-salary 0.97 1.00 0.98 30 other 1.00 0.99 1.00 106 accuracy - - 0.99 136 macro-avg 0.98 1.00 0.99 136 weighted-avg 0.99 0.99 0.99 136 ``` --- layout: model title: Legal Exercise of option Clause Binary Classifier author: John Snow Labs name: legclf_exercise_of_option_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `exercise-of-option` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `exercise-of-option` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_exercise_of_option_clause_en_1.0.0_3.2_1660122426759.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_exercise_of_option_clause_en_1.0.0_3.2_1660122426759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_exercise_of_option_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[exercise-of-option]| |[other]| |[other]| |[exercise-of-option]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_exercise_of_option_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support exercise-of-option 0.97 0.90 0.93 39 other 0.96 0.99 0.97 91 accuracy - - 0.96 130 macro-avg 0.96 0.94 0.95 130 weighted-avg 0.96 0.96 0.96 130 ``` --- layout: model title: English BertForMaskedLM Cased model (from philschmid) author: John Snow Labs name: bert_embeddings_fin_pretrain_yiyanghkust date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `finbert-pretrain-yiyanghkust` is a English model originally trained by `philschmid`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_fin_pretrain_yiyanghkust_en_4.2.4_3.0_1670022094241.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_fin_pretrain_yiyanghkust_en_4.2.4_3.0_1670022094241.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_fin_pretrain_yiyanghkust","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_fin_pretrain_yiyanghkust","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_fin_pretrain_yiyanghkust| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|412.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/philschmid/finbert-pretrain-yiyanghkust - https://arxiv.org/abs/2006.08097 --- layout: model title: English BertForQuestionAnswering model (from Salesforce) author: John Snow Labs name: bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qaconv-bert-large-uncased-whole-word-masking-squad2` is a English model orginally trained by `Salesforce`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2_en_4.0.0_3.0_1654189117017.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2_en_4.0.0_3.0_1654189117017.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_uncased.by_Salesforce").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_qaconv_bert_large_uncased_whole_word_masking_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Salesforce/qaconv-bert-large-uncased-whole-word-masking-squad2 --- layout: model title: Thai DistilBERT Embeddings author: John Snow Labs name: distilbert_embeddings_distilbert_base_th_cased date: 2022-04-12 tags: [distilbert, embeddings, th, open_source] task: Embeddings language: th edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-th-cased` is a Thai model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_th_cased_th_3.4.2_3.0_1649784017728.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_th_cased_th_3.4.2_3.0_1649784017728.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_th_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|th| |Size:|185.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-th-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate Efik to English Pipeline author: John Snow Labs name: translate_efi_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, efi, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `efi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_efi_en_xx_2.7.0_2.4_1609699653371.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_efi_en_xx_2.7.0_2.4_1609699653371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_efi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_efi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.efi.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_efi_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: RxNorm Scd ChunkResolver author: John Snow Labs name: chunkresolve_rxnorm_scd_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-07-27 task: Entity Resolution edition: Healthcare NLP 2.5.1 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities RxNorm Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_scd_clinical_en_2.5.1_2.4_1595813884363.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_rxnorm_scd_clinical_en_2.5.1_2.4_1595813884363.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... rxnormResolver = ChunkEntityResolverModel()\ .pretrained('chunkresolve_rxnorm_scd_clinical', 'en', "clinical/models")\ .setEnableLevenshtein(True)\ .setNeighbours(200).setAlternatives(5).setDistanceWeights([3,3,2,0,0,7])\ .setInputCols(['token', 'chunk_embs_drug'])\ .setOutputCol('rxnorm_resolution')\ pipeline_rxnorm = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, rxnormResolver]) model = pipeline_rxnorm.fit(spark.createDataFrame([['']]).toDF("text")) results = model.transform(data) ``` ```scala ... val rxnormResolver = ChunkEntityResolverModel() .pretrained('chunkresolve_rxnorm_scd_clinical', 'en', "clinical/models") .setEnableLevenshtein(True) .setNeighbours(200).setAlternatives(5).setDistanceWeights(Array(3,3,2,0,0,7)) .setInputCols('token', 'chunk_embs_drug') .setOutputCol('rxnorm_resolution') val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, rxnormResolver)) val result = pipeline.fit(Seq.empty[String]).transform(data) ```

{:.h2_title} ## Results ```bash | coords | chunk | entity | rxnorm_opts | |--------------|-------------|-----------|-----------------------------------------------------------------------------------------| | 3::278::287 | creatinine | DrugChem | [(849628, Creatinine 800 MG Oral Capsule), (252180, Urea 10 MG/ML Topical Lotion), ...] | | 7::83::93 | cholesterol | DrugChem | [(2104173, beta Sitosterol 35 MG Oral Tablet), (832876, phytosterol esters 500 MG O...] | | 10::397::406 | creatinine | DrugChem | [(849628, Creatinine 800 MG Oral Capsule), (252180, Urea 10 MG/ML Topical Lotion), ...] | ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------------| | Name: | chunkresolve_rxnorm_scd_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.5.1+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on December 2019 RxNorm Clinical Drugs (TTY=SCD) ontology graph with `embeddings_clinical` https://www.nlm.nih.gov/pubs/techbull/nd19/brief/nd19_rxnorm_december_2019_release.html --- layout: model title: English BertForQuestionAnswering model (from JAlexis) author: John Snow Labs name: bert_qa_PruebaBert date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `PruebaBert` is a English model orginally trained by `JAlexis`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_PruebaBert_en_4.0.0_3.0_1654179023524.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_PruebaBert_en_4.0.0_3.0_1654179023524.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_PruebaBert","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_PruebaBert","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.cord19.prueba_bert.by_JAlexis").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_PruebaBert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/JAlexis/PruebaBert --- layout: model title: Pipeline to Detect Clinical Entities author: John Snow Labs name: ner_jsl_enriched_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_enriched](https://nlp.johnsnowlabs.com/2021/10/22/ner_jsl_enriched_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_pipeline_en_3.4.1_3.0_1647869433247.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_pipeline_en_3.4.1_3.0_1647869433247.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_jsl_enriched_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("ner_jsl_enriched_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_enriched.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | chunk | begin | end | entity | |---:|:------------------------------------------|--------:|------:|:-----------------------------| | 0 | 21-day-old | 17 | 26 | Age | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | | 2 | male | 38 | 41 | Gender | | 3 | 2 days | 52 | 57 | Duration | | 4 | congestion | 62 | 71 | Symptom | | 5 | mom | 75 | 77 | Gender | | 6 | suctioning yellow discharge | 88 | 114 | Symptom | | 7 | nares | 135 | 139 | External_body_part_or_region | | 8 | she | 147 | 149 | Gender | | 9 | mild | 168 | 171 | Modifier | | 10 | problems with his breathing while feeding | 173 | 213 | Symptom | | 11 | perioral cyanosis | 237 | 253 | Symptom | | 12 | retractions | 258 | 268 | Symptom | | 13 | One day ago | 272 | 282 | RelativeDate | | 14 | mom | 285 | 287 | Gender | | 15 | tactile temperature | 304 | 322 | Symptom | | 16 | Tylenol | 345 | 351 | Drug_BrandName | | 17 | Baby | 354 | 357 | Age | | 18 | decreased p.o. intake | 377 | 397 | Symptom | | 19 | His | 400 | 402 | Gender | | 20 | q.2h | 450 | 453 | Frequency | | 21 | 5 to 10 minutes | 459 | 473 | Duration | | 22 | his | 488 | 490 | Gender | | 23 | respiratory congestion | 492 | 513 | Symptom | | 24 | He | 516 | 517 | Gender | | 25 | tired | 550 | 554 | Symptom | | 26 | fussy | 569 | 573 | Symptom | | 27 | over the past 2 days | 575 | 594 | RelativeDate | | 28 | albuterol | 637 | 645 | Drug_Ingredient | | 29 | ER | 671 | 672 | Clinical_Dept | | 30 | His | 675 | 677 | Gender | | 31 | urine output has also decreased | 679 | 709 | Symptom | | 32 | he | 721 | 722 | Gender | | 33 | per 24 hours | 760 | 771 | Frequency | | 34 | he | 778 | 779 | Gender | | 35 | per 24 hours | 807 | 818 | Frequency | | 36 | Mom | 821 | 823 | Gender | | 37 | diarrhea | 836 | 843 | Symptom | | 38 | His | 846 | 848 | Gender | | 39 | bowel | 850 | 854 | Internal_organ_or_component | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_enriched_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English image_classifier_vit_string_instrument_detector ViTForImageClassification from rexoscare author: John Snow Labs name: image_classifier_vit_string_instrument_detector date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_string_instrument_detector` is a English model originally trained by rexoscare. ## Predicted Entities `Banjo`, `Guitar`, `Mandolin`, `Ukulele` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_string_instrument_detector_en_4.1.0_3.0_1660169802968.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_string_instrument_detector_en_4.1.0_3.0_1660169802968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_string_instrument_detector", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_string_instrument_detector", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_string_instrument_detector| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_job_all_903929564 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-job_all-903929564` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Job`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_job_all_903929564_en_4.3.1_3.0_1678134005364.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_job_all_903929564_en_4.3.1_3.0_1678134005364.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_job_all_903929564","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_job_all_903929564","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_job_all_903929564| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-job_all-903929564 --- layout: model title: Translate English to Fijian Pipeline author: John Snow Labs name: translate_en_fj date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, fj, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `fj` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_fj_xx_2.7.0_2.4_1609688726797.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_fj_xx_2.7.0_2.4_1609688726797.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_fj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_fj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.fj').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_fj| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Illegality Clause Binary Classifier author: John Snow Labs name: legclf_illegality_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `illegality` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `illegality` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_illegality_clause_en_1.0.0_3.2_1660123594068.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_illegality_clause_en_1.0.0_3.2_1660123594068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_illegality_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[illegality]| |[other]| |[other]| |[illegality]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_illegality_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support illegality 0.95 0.95 0.95 22 other 0.98 0.98 0.98 56 accuracy - - 0.97 78 macro-avg 0.97 0.97 0.97 78 weighted-avg 0.97 0.97 0.97 78 ``` --- layout: model title: Spanish Part of Speech Tagger (Large) author: John Snow Labs name: roberta_pos_roberta_large_bne_capitel_pos date: 2022-05-03 tags: [roberta, pos, part_of_speech, es, open_source] task: Part of Speech Tagging language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-large-bne-capitel-pos` is a Spanish model orginally trained by `PlanTL-GOB-ES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_large_bne_capitel_pos_es_3.4.2_3.0_1651595953421.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_pos_roberta_large_bne_capitel_pos_es_3.4.2_3.0_1651595953421.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_roberta_large_bne_capitel_pos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-large-bne-capitel-pos - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://sites.google.com/view/capitel2020 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Marathi ALBERT Embeddings (v2) author: John Snow Labs name: albert_embeddings_marathi_albert_v2 date: 2022-04-14 tags: [albert, embeddings, mr, open_source] task: Embeddings language: mr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `marathi-albert-v2` is a Marathi model orginally trained by `l3cube-pune`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_marathi_albert_v2_mr_3.4.2_3.0_1649954249563.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_marathi_albert_v2_mr_3.4.2_3.0_1649954249563.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_marathi_albert_v2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|mr| |Size:|128.3 MB| |Case sensitive:|false| ## References - https://huggingface.co/l3cube-pune/marathi-albert-v2 - https://github.com/l3cube-pune/MarathiNLP - https://arxiv.org/abs/2202.01159 --- layout: model title: English asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml TFWav2Vec2ForCTC from arbml author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml` is a English model originally trained by arbml. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml_en_4.2.0_3.0_1664096871274.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml_en_4.2.0_3.0_1664096871274.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_arabic_egyptian_by_arbml| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForMaskedLM Large Uncased model author: John Snow Labs name: bert_embeddings_large_uncased_whole_word_masking date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking` is a English model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_uncased_whole_word_masking_en_4.2.4_3.0_1670020551279.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_uncased_whole_word_masking_en_4.2.4_3.0_1670020551279.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_uncased_whole_word_masking","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_uncased_whole_word_masking","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_uncased_whole_word_masking| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/bert-large-uncased-whole-word-masking - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10_en_4.0.0_3.0_1655732897981.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10_en_4.0.0_3.0_1655732897981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_512d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|432.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-10 --- layout: model title: Arabic Part of Speech Tagger (Modern Standard Arabic-MSA POS, Modern Standard Arabic-MSA, Dialectal Arabic-DA, and Classical Arabic-CA) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_mix_pos_msa date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-mix-pos-msa` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_mix_pos_msa_ar_3.4.2_3.0_1650993325530.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_mix_pos_msa_ar_3.4.2_3.0_1650993325530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_mix_pos_msa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-mix-pos-msa - https://dl.acm.org/doi/pdf/10.5555/1621804.1621808 - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: English ElectraForQuestionAnswering model (from rowan1224) Squad author: John Snow Labs name: electra_qa_squad_slp date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-squad-slp` is a English model originally trained by `rowan1224`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_squad_slp_en_4.0.0_3.0_1655921330994.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_squad_slp_en_4.0.0_3.0_1655921330994.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_squad_slp","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_squad_slp","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_squad_slp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rowan1224/electra-squad-slp --- layout: model title: English image_classifier_vit_oz_fauna ViTForImageClassification from lewtun author: John Snow Labs name: image_classifier_vit_oz_fauna date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_oz_fauna` is a English model originally trained by lewtun. ## Predicted Entities `koala`, `kookaburra`, `dingo`, `possum`, `tasmanian devil` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_oz_fauna_en_4.1.0_3.0_1660170193715.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_oz_fauna_en_4.1.0_3.0_1660170193715.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_oz_fauna", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_oz_fauna", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_oz_fauna| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from manishiitg) author: John Snow Labs name: bert_qa_spanbert_large_recruit_qa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-large-recruit-qa` is a English model orginally trained by `manishiitg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_large_recruit_qa_en_4.0.0_3.0_1654191827632.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_large_recruit_qa_en_4.0.0_3.0_1654191827632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_large_recruit_qa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_large_recruit_qa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.span_bert.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_large_recruit_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/manishiitg/spanbert-large-recruit-qa --- layout: model title: Public Health Mention Classifier (PHS-BERT) author: John Snow Labs name: classifierdl_health_mentions date: 2022-07-25 tags: [public_health, health, mention, en, licensed, classification] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [PHS-BERT](https://arxiv.org/abs/2204.04521) based classifier that can classify public health mentions in social media text. Mentions are classified into three labels about personal health situation, figurative mention and other mentions. More detailed information about classes as follows: `health_mention`: The text contains a health mention that specifically indicating someone's health situation. This means someone has a certain disease or symptoms including death. e.g.; *My PCR test is positive. I have a severe joint pain, mucsle pain and headache right now.* `other_mention`: The text contains a health mention; however does not states a spesific person's situation. General health mentions like informative mentions, discussion about disease etc. e.g.; *Aluminum is a light metal that causes dementia and Alzheimer's disease.* `figurative_mention`: The text mention specific disease or symptom but it is used metaphorically, does not contain health-related information. e.g.; *I don't wanna fall in love. If I ever did that, I think I'd have a heart attack.* ## Predicted Entities `figurative_mention`, `other_mention`, `health_mention` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_health_mentions_en_4.0.0_3.0_1658759311177.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_health_mentions_en_4.0.0_3.0_1658759311177.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") bert_embeddings = BertEmbeddings.pretrained("bert_embeddings_phs_bert", "en", "public/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ embeddingsSentence = SentenceEmbeddings() \ .setInputCols(["sentence", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifierdl = ClassifierDLModel.pretrained('classifierdl_health_mentions', 'en', 'clinical/models')\ .setInputCols(['sentence', 'token', 'sentence_embeddings'])\ .setOutputCol('class') clf_pipeline = Pipeline( stages = [ document_assembler, tokenizer, bert_embeddings, embeddingsSentence, classifierdl ]) data = spark.createDataFrame([["I feel a bit drowsy & have a little blurred vision after taking an insulin."]]).toDF("text") result = clf_pipeline.fit(data).transform(data) result.select("text", "class.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("word_embeddings") val sentence_embeddings = SentenceEmbeddings() .setInputCols(Array("sentence", "word_embeddings")) .setOutputCol("sentence_embeddings") .setPoolingStrategy("AVERAGE") val classifier = ClassifierDLModel.pretrained("classifierdl_health_mentions", "en", "clinical/models") .setInputCols(Array("sentence", "token", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, sentence_embeddings, classifier)) val data = Seq("I feel a bit drowsy & have a little blurred vision after taking an insulin.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.health").predict("""I feel a bit drowsy & have a little blurred vision after taking an insulin.""") ```

## Results ```bash +---------------------------------------------------------------------------+----------------+ |text |class | +---------------------------------------------------------------------------+----------------+ |I feel a bit drowsy & have a little blurred vision after taking an insulin.|[health_mention]| +---------------------------------------------------------------------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_health_mentions| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|24.1 MB| ## References Curated from several academic and in-house datasets. ## Benchmarking ```bash precision recall f1-score support health_mention 0.77 0.83 0.80 1375 other_mention 0.84 0.81 0.83 2102 figurative_mention 0.79 0.78 0.79 1412 accuracy - - 0.81 4889 macro-avg 0.80 0.81 0.80 4889 weighted-avg 0.81 0.81 0.81 4889 ``` --- layout: model title: Entity Recognizer LG author: John Snow Labs name: entity_recognizer_lg date: 2022-06-25 tags: [fi, open_source] task: Named Entity Recognition language: fi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_fi_4.0.0_3.0_1656125094034.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_lg_fi_4.0.0_3.0_1656125094034.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_lg| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|2.5 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: English image_classifier_vit_Test_Model ViTForImageClassification from Nonem100 author: John Snow Labs name: image_classifier_vit_Test_Model date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Test_Model` is a English model originally trained by Nonem100. ## Predicted Entities `nachos`, `popcorn`, `cotton candy`, `hot dog`, `hamburger` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Test_Model_en_4.1.0_3.0_1660169509995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Test_Model_en_4.1.0_3.0_1660169509995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Test_Model", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Test_Model", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Test_Model| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: ICD10CM Musculoskeletal Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_musculoskeletal_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_musculoskeletal_clinical_en_3.0.0_3.0_1617355429847.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_musculoskeletal_clinical_en_3.0.0_3.0_1617355429847.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... muscu_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_musculoskeletal_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, muscu_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val muscu_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_musculoskeletal_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, muscu_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash chunk entity icd10_muscu_description icd10_muscu_code 0 a cold, cough PROBLEM Postprocedural hemorrhage of a musculoskeletal... M96831 1 runny nose PROBLEM Acquired deformity of nose M950 2 fever PROBLEM Periodic fever syndromes M041 3 difficulty breathing PROBLEM Other dentofacial functional abnormalities M2659 4 her cough PROBLEM Cervicalgia M542 5 physical exam TEST Pathological fracture, unspecified toe(s), seq... M84479S 6 fairly congested PROBLEM Synovial hypertrophy, not elsewhere classified... M67262 7 Amoxil TREATMENT Torticollis M436 8 Aldex TREATMENT Other soft tissue disorders related to use, ov... M7088 9 difficulty breathing PROBLEM Other dentofacial functional abnormalities M2659 10 more congested PROBLEM Pain in unspecified ankle and joints of unspec... M25579 11 trouble sleeping PROBLEM Low back pain M545 12 congestion PROBLEM Progressive systemic sclerosis M340 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_icd10cm_musculoskeletal_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[icd10cm]| |Language:|en| --- layout: model title: Sentence Detection in Somali Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [open_source, sentence_detection, so] task: Sentence Detection language: so edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_so_3.2.0_3.0_1630321968392.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_so_3.2.0_3.0_1630321968392.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "so") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""Raadinta il weyn oo ka mid ah cutubyada akhriska Ingiriisiga? Waxaad timid meeshii saxda ahayd Sida laga soo xigtay daraasad dhowaan la sameeyay, caadadii wax -akhriska ee dhallinyarada maanta ayaa si degdeg ah hoos ugu dhacaysa. Waxay diiradda saari karin cutubka akhriska Ingiriisiga ee la siiyay wax ka badan dhowr ilbiriqsi! Sidoo kale, akhrintu waxay ahayd qayb muhiim ah oo ka mid ah dhammaan imtixaannada tartanka. Haddaba, sidee u hagaajin kartaa xirfadahaaga akhriska? Jawaabta su'aashan dhab ahaantii waa su'aal kale: Waa maxay isticmaalka xirfadaha akhriska? Ujeeddada ugu weyn ee wax -akhrisku waa 'macno samayn'.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "so") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("Raadinta il weyn oo ka mid ah cutubyada akhriska Ingiriisiga? Waxaad timid meeshii saxda ahayd Sida laga soo xigtay daraasad dhowaan la sameeyay, caadadii wax -akhriska ee dhallinyarada maanta ayaa si degdeg ah hoos ugu dhacaysa. Waxay diiradda saari karin cutubka akhriska Ingiriisiga ee la siiyay wax ka badan dhowr ilbiriqsi! Sidoo kale, akhrintu waxay ahayd qayb muhiim ah oo ka mid ah dhammaan imtixaannada tartanka. Haddaba, sidee u hagaajin kartaa xirfadahaaga akhriska? Jawaabta su'aashan dhab ahaantii waa su'aal kale: Waa maxay isticmaalka xirfadaha akhriska? Ujeeddada ugu weyn ee wax -akhrisku waa 'macno samayn'.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('so.sentence_detector').predict("Raadinta il weyn oo ka mid ah cutubyada akhriska Ingiriisiga? Waxaad timid meeshii saxda ahayd Sida laga soo xigtay daraasad dhowaan la sameeyay, caadadii wax -akhriska ee dhallinyarada maanta ayaa si degdeg ah hoos ugu dhacaysa. Waxay diiradda saari karin cutubka akhriska Ingiriisiga ee la siiyay wax ka badan dhowr ilbiriqsi! Sidoo kale, akhrintu waxay ahayd qayb muhiim ah oo ka mid ah dhammaan imtixaannada tartanka. Haddaba, sidee u hagaajin kartaa xirfadahaaga akhriska? Jawaabta su'aashan dhab ahaantii waa su'aal kale: Waa maxay isticmaalka xirfadaha akhriska? Ujeeddada ugu weyn ee wax -akhrisku waa 'macno samayn'.", output_level ='sentence') ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_posology_large_biobert_pipeline", "en", "clinical/models") pipeline.fullAnnotate('The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.') ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_posology_large_biobert_pipeline", "en", "clinical/models") pipeline.fullAnnotate("The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology_biobert_large.pipeline").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunks |entities | +--------------+---------+ |Bactrim |DRUG | |for 14 days |DURATION | |Fragmin |DRUG | |5000 units |STRENGTH | |subcutaneously|ROUTE | |daily |FREQUENCY| |Xenaderm |DRUG | |topically |ROUTE | |b.i.d |FREQUENCY| |Lantus |DRUG | |40 units |STRENGTH | |subcutaneously|ROUTE | |at bedtime |FREQUENCY| |OxyContin |DRUG | |30 mg |STRENGTH | |p.o |ROUTE | |folic acid |DRUG | |1 mg |STRENGTH | |daily |FREQUENCY| |levothyroxine |DRUG | +--------------+---------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_large_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|421.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Chinese T5ForConditionalGeneration Base Cased model (from Langboat) author: John Snow Labs name: t5_mengzi_base date: 2023-01-30 tags: [zh, open_source, t5, tensorflow] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mengzi-t5-base` is a Chinese model originally trained by `Langboat`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_mengzi_base_zh_4.3.0_3.0_1675105125486.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_mengzi_base_zh_4.3.0_3.0_1675105125486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_mengzi_base","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_mengzi_base","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_mengzi_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|1.0 GB| ## References - https://huggingface.co/Langboat/mengzi-t5-base - https://arxiv.org/abs/2110.06696 --- layout: model title: Detect Entities Related to Cancer Therapies author: John Snow Labs name: ner_oncology_therapy date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner, treatment] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to oncology therapies using granular labels, including mentions of treatments, posology information and line of therapy. Definitions of Predicted Entities: - `Cancer_Surgery`: Terms that indicate surgery as a form of cancer treatment. - `Chemotherapy`: Mentions of chemotherapy drugs, or unspecific words such as "chemotherapy". - `Cycle_Count`: The total number of cycles being administered of an oncological therapy (e.g. "5 cycles"). - `Cycle_Day`: References to the day of the cycle of oncological therapy (e.g. "day 5"). - `Cycle_Number`: The number of the cycle of an oncological therapy that is being applied (e.g. "third cycle"). - `Dosage`: The quantity prescribed by the physician for an active ingredient. - `Duration`: Words indicating the duration of a treatment (e.g. "for 2 weeks"). - `Frequency`: Words indicating the frequency of treatment administration (e.g. "daily" or "bid"). - `Hormonal_Therapy`: Mentions of hormonal drugs used to treat cancer, or unspecific words such as "hormonal therapy". - `Immunotherapy`: Mentions of immunotherapy drugs, or unspecific words such as "immunotherapy". - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Radiotherapy`: Terms that indicate the use of Radiotherapy. - `Radiation_Dose`: Dose used in radiotherapy. - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Route`: Words indicating the type of administration route (such as "PO" or "transdermal"). - `Targeted_Therapy`: Mentions of targeted therapy drugs, or unspecific words such as "targeted therapy". - `Unspecific_Therapy`: Terms that indicate a known cancer therapy but that is not specific to any other therapy entity (e.g. "chemoradiotherapy" or "adjuvant therapy"). ## Predicted Entities `Cancer_Surgery`, `Chemotherapy`, `Cycle_Count`, `Cycle_Day`, `Cycle_Number`, `Dosage`, `Duration`, `Frequency`, `Hormonal_Therapy`, `Immunotherapy`, `Line_Of_Therapy`, `Radiotherapy`, `Radiation_Dose`, `Response_To_Treatment`, `Route`, `Targeted_Therapy`, `Unspecific_Therapy` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_en_4.0.0_3.0_1666718855759.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_en_4.0.0_3.0_1666718855759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-------------------------------|:----------------------| | mastectomy | Cancer_Surgery | | axillary lymph node dissection | Cancer_Surgery | | radiotherapy | Radiotherapy | | recurred | Response_To_Treatment | | adriamycin | Chemotherapy | | 60 mg/m2 | Dosage | | cyclophosphamide | Chemotherapy | | 600 mg/m2 | Dosage | | six courses | Cycle_Count | | first line | Line_Of_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_therapy| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.4 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Cycle_Number 78 41 19 97 0.66 0.80 0.72 Response_To_Treatment 451 205 145 596 0.69 0.76 0.72 Cycle_Count 210 75 20 230 0.74 0.91 0.82 Unspecific_Therapy 189 76 89 278 0.71 0.68 0.70 Chemotherapy 831 87 48 879 0.91 0.95 0.92 Targeted_Therapy 194 28 34 228 0.87 0.85 0.86 Radiotherapy 279 35 31 310 0.89 0.90 0.89 Cancer_Surgery 720 192 99 819 0.79 0.88 0.83 Line_Of_Therapy 95 6 11 106 0.94 0.90 0.92 Hormonal_Therapy 170 6 15 185 0.97 0.92 0.94 Immunotherapy 96 17 32 128 0.85 0.75 0.80 Cycle_Day 205 38 43 248 0.84 0.83 0.84 Frequency 363 33 64 427 0.92 0.85 0.88 Route 93 6 20 113 0.94 0.82 0.88 Duration 527 102 234 761 0.84 0.69 0.76 Dosage 959 63 101 1060 0.94 0.90 0.92 Radiation_Dose 106 12 20 126 0.90 0.84 0.87 macro_avg 5566 1022 1025 6591 0.85 0.84 0.84 micro_avg 5566 1022 1025 6591 0.85 0.84 0.84 ``` --- layout: model title: Clinical Deidentification (French) author: John Snow Labs name: clinical_deidentification date: 2023-06-13 tags: [deid, fr, licensed] task: De-identification language: fr edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts in French. The pipeline can mask and obfuscate the following entities: `DATE`, `AGE`, `SEX`, `PROFESSION`, `ORGANIZATION`, `PHONE`, `E-MAIL`, `ZIP`, `STREET`, `CITY`, `COUNTRY`, `PATIENT`, `DOCTOR`, `HOSPITAL`, `MEDICALRECORD`, `SSN`, `IDNUM`, `ACCOUNT`, `PLATE`, `USERNAME`, `URL`, and `IPADDR`. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_fr_4.4.4_3.2_1686663918009.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_fr_4.4.4_3.2_1686663918009.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "fr", "clinical/models") sample = """COMPTE-RENDU D'HOSPITALISATION PRENOM : Jean NOM : Dubois NUMÉRO DE SÉCURITÉ SOCIALE : 1780160471058 ADRESSE : 18 Avenue Matabiau VILLE : Grenoble CODE POSTAL : 38000 DATE DE NAISSANCE : 03/03/1946 Âge : 70 ans Sexe : H COURRIEL : jdubois@hotmail.fr DATE D'ADMISSION : 12/12/2016 MÉDÉCIN : Dr Michel Renaud RAPPORT CLINIQUE : 70 ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. Il nous a été adressé car il présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : Dre Marie Breton - Centre Hospitalier de Bellevue Service D'Endocrinologie et de Nutrition - Rue Paulin Bussières, 38000 Grenoble COURRIEL : mariebreton@chb.fr """ result = deid_pipeline.annotate(sample) ``` ```scala val deid_pipeline = new PretrainedPipeline("clinical_deidentification", "fr", "clinical/models") sample = "COMPTE-RENDU D'HOSPITALISATION PRENOM : Jean NOM : Dubois NUMÉRO DE SÉCURITÉ SOCIALE : 1780160471058 ADRESSE : 18 Avenue Matabiau VILLE : Grenoble CODE POSTAL : 38000 DATE DE NAISSANCE : 03/03/1946 Âge : 70 ans Sexe : H COURRIEL : jdubois@hotmail.fr DATE D'ADMISSION : 12/12/2016 MÉDÉCIN : Dr Michel Renaud RAPPORT CLINIQUE : 70 ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. Il nous a été adressé car il présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : Dre Marie Breton - Centre Hospitalier de Bellevue Service D'Endocrinologie et de Nutrition - Rue Paulin Bussières, 38000 Grenoble COURRIEL : mariebreton@chb.fr " val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("fr.deid_obfuscated").predict("""COMPTE-RENDU D'HOSPITALISATION PRENOM : Jean NOM : Dubois NUMÉRO DE SÉCURITÉ SOCIALE : 1780160471058 ADRESSE : 18 Avenue Matabiau VILLE : Grenoble CODE POSTAL : 38000 DATE DE NAISSANCE : 03/03/1946 Âge : 70 ans Sexe : H COURRIEL : jdubois@hotmail.fr DATE D'ADMISSION : 12/12/2016 MÉDÉCIN : Dr Michel Renaud RAPPORT CLINIQUE : 70 ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. Il nous a été adressé car il présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : Dre Marie Breton - Centre Hospitalier de Bellevue Service D'Endocrinologie et de Nutrition - Rue Paulin Bussières, 38000 Grenoble COURRIEL : mariebreton@chb.fr """) ```

## Results ```bash Results Masked with entity labels ------------------------------ COMPTE-RENDU D'HOSPITALISATION PRENOM : NOM : NUMÉRO DE SÉCURITÉ SOCIALE : ADRESSE : VILLE : CODE POSTAL : DATE DE NAISSANCE : Âge : Sexe : COURRIEL : DATE D'ADMISSION : MÉDÉCIN : RAPPORT CLINIQUE : ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. nous a été adressé car présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : - Service D'Endocrinologie et de Nutrition - , COURRIEL : Masked with chars ------------------------------ COMPTE-RENDU D'HOSPITALISATION PRENOM : [**] NOM : [****] NUMÉRO DE SÉCURITÉ SOCIALE : [***********] ADRESSE : [****************] VILLE : [******] CODE POSTAL : [***] DATE DE NAISSANCE : [********] Âge : [****] Sexe : * COURRIEL : [****************] DATE D'ADMISSION : [********] MÉDÉCIN : [**************] RAPPORT CLINIQUE : ** ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. ** nous a été adressé car ** présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : [**************] - [****************************] Service D'Endocrinologie et de Nutrition - [******************], [***] [******] COURRIEL : [****************] Masked with fixed length chars ------------------------------ COMPTE-RENDU D'HOSPITALISATION PRENOM : **** NOM : **** NUMÉRO DE SÉCURITÉ SOCIALE : **** ADRESSE : **** VILLE : **** CODE POSTAL : **** DATE DE NAISSANCE : **** Âge : **** Sexe : **** COURRIEL : **** DATE D'ADMISSION : **** MÉDÉCIN : **** RAPPORT CLINIQUE : **** ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. **** nous a été adressé car **** présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : **** - **** Service D'Endocrinologie et de Nutrition - ****, **** **** COURRIEL : **** Obfuscated ------------------------------ COMPTE-RENDU D'HOSPITALISATION PRENOM : Mme Ollivier NOM : Mme Traore NUMÉRO DE SÉCURITÉ SOCIALE : 164033818514436 ADRESSE : 731, boulevard de Legrand VILLE : Sainte Antoine CODE POSTAL : 37443 DATE DE NAISSANCE : 18/03/1946 Âge : 46 Sexe : Femme COURRIEL : georgeslemonnier@live.com DATE D'ADMISSION : 10/01/2017 MÉDÉCIN : Pr. Manon Dupuy RAPPORT CLINIQUE : 26 ans, retraité, sans allergie médicamenteuse connue, qui présente comme antécédents : ancien accident du travail avec fractures vertébrales et des côtes ; opéré de la maladie de Dupuytren à la main droite et d'un pontage ilio-fémoral gauche ; diabète de type II, hypercholestérolémie et hyperuricémie ; alcoolisme actif, fume 20 cigarettes / jour. Homme nous a été adressé car Homme présentait une hématurie macroscopique postmictionnelle à une occasion et une microhématurie persistante par la suite, avec une miction normale. L'examen physique a montré un bon état général, avec un abdomen et des organes génitaux normaux ; le toucher rectal était compatible avec un adénome de la prostate de grade I/IV. L'analyse d'urine a montré 4 globules rouges/champ et 0-5 leucocytes/champ ; le reste du sédiment était normal. Hémogramme normal ; la biochimie a montré une glycémie de 169 mg/dl et des triglycérides de 456 mg/dl ; les fonctions hépatiques et rénales étaient normales. PSA de 1,16 ng/ml. ADDRESSÉ À : Dr Tristan-Gilbert Poulain - CENTRE HOSPITALIER D'ORTHEZ Service D'Endocrinologie et de Nutrition - 6, avenue Pages, 37443 Sainte Antoine COURRIEL : massecatherine@bouygtel.fr {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|fr| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - RegexMatcherModel - RegexMatcherModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English DistilBertForQuestionAnswering Cased model (from abhilash1910) author: John Snow Labs name: distilbert_qa_squadv1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-squadv1` is a English model originally trained by `abhilash1910`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squadv1_en_4.3.0_3.0_1672774380902.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squadv1_en_4.3.0_3.0_1672774380902.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squadv1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squadv1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/abhilash1910/distilbert-squadv1 --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_tiny_3_finetuned_squadv2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-tiny-3-finetuned-squadv2` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_tiny_3_finetuned_squadv2_en_4.0.0_3.0_1654184985253.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_tiny_3_finetuned_squadv2_en_4.0.0_3.0_1654184985253.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_tiny_3_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_tiny_3_finetuned_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.tiny_v3.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_tiny_3_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|21.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-tiny-3-finetuned-squadv2 --- layout: model title: Legal Criminal Law Document Classifier (EURLEX) author: John Snow Labs name: legclf_criminal_law_bert date: 2023-03-06 tags: [en, legal, classification, clauses, criminal_law, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_criminal_law_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Criminal_Law or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Criminal_Law`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_criminal_law_bert_en_1.0.0_3.0_1678111769534.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_criminal_law_bert_en_1.0.0_3.0_1678111769534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_criminal_law_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Criminal_Law]| |[Other]| |[Other]| |[Criminal_Law]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_criminal_law_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Criminal_Law 0.88 0.86 0.87 74 Other 0.84 0.85 0.85 61 accuracy - - 0.86 135 macro-avg 0.86 0.86 0.86 135 weighted-avg 0.86 0.86 0.86 135 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from anurag0077) Squad author: John Snow Labs name: distilbert_qa_anurag0077_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `anurag0077`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_anurag0077_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725004904.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_anurag0077_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725004904.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anurag0077_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anurag0077_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased_v3.by_anurag0077").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_anurag0077_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anurag0077/distilbert-base-uncased-finetuned-squad --- layout: model title: Detect Oncology Entities author: John Snow Labs name: ner_oncology_wip date: 2022-07-25 tags: [licensed, english, clinical, ner, oncology, cancer, biomarker, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.5.0 spark_version: 3.0 supported: true published: false annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained named entity recognition (NER) model is a deep learning model for detecting entities related to cancer diagnosis (such as staging, grade or mentiones of metastasis), cancer treatments (chemotherapy, targeted therapy or surgical procedures, among other), and oncological tests (pathology tests, biomarkers, oncogenes, etc). The model was trained using `MedicalNerApproach` annotator that allows to train generic NER models based on Neural Networks. ## Predicted Entities `Gender`, `Age`, `Race_Ethnicity`, `Date`, `Cancer_Dx`, `Metastasis`, `Invasion`, `Histological_Type`, `Grade`, `Tumor_Finding`, `Staging`, `Tumor_Size`, `Oncogene`, `Biomarker`, `Biomarker_Result`, `Performance_Status`, `Pathology_Test`, `Pathology_Result`, `Smoking_Status`, `Anatomical_Site`, `Direction`, `Site_Lymph_Node`, `Chemotherapy`, `Immunotherapy`, `Targeted_Therapy`, `Hormonal_Therapy`, `Unspecific_Therapy`, `Radiotherapy`, `Cancer_Surgery`, `Line_Of_Therapy`, `Response_To_Treatment`, `Radiation_Dose`, `Duration`, `Frequency`, `Cycle_Number`, `Cycle_Day`, `Dosage`, `Route`, `Relative_Date`, `Imaging_Test` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_wip_en_3.5.0_3.0_1658771306053.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_wip_en_3.5.0_3.0_1658771306053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------+-----+---+-----------------+ | chunk|begin|end| ner_label| +----------------------------------------+-----+---+-----------------+ | She| 0| 2| Gender| | breast| 40| 45| Anatomical_Site| | lump| 47| 50| Tumor_Finding| | she| 57| 59| Gender| | for the past few months| 73| 95| Duration| | breast cancer| 131|143| Cancer_Dx| | She| 146|148| Gender| | right| 173|177| Direction| | wide local excision| 179|197| Cancer_Surgery| | mass| 206|209| Tumor_Finding| | axillary clearance| 215|232| Anatomical_Site| | Histology| 235|243| Pathology_Test| | oestrogen receptor| 267|284| Biomarker| | positive| 286|293| Biomarker_Result| |human epidermal growth factor receptor 2| 296|335| Oncogene| | negative| 337|344| Biomarker_Result| | ductal| 346|351|Histological_Type| | carcinoma involving 12| 353|374| Cancer_Dx| | 14 axillary nodes| 379|395| Site_Lymph_Node| +----------------------------------------+-----+---+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_wip| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|965.8 KB| ## References Trained on case reports sampled from PubMed, and annotated in-house. ## Benchmarking ```bash label tp fp fn total precision recall f1 Histological_Type 181.0 48.0 48.0 229.0 0.790393 0.790393 0.790393 Anatomical_Site 1166.0 401.0 208.0 1374.0 0.744097 0.848617 0.792928 Direction 391.0 75.0 159.0 550.0 0.839056 0.710909 0.769685 Staging 187.0 28.0 32.0 219.0 0.869767 0.853881 0.861751 Imaging_Test 959.0 119.0 120.0 1079.0 0.889610 0.888786 0.889198 Cycle_Number 162.0 43.0 25.0 187.0 0.790244 0.866310 0.826531 Tumor_Finding 446.0 62.0 96.0 542.0 0.877953 0.822878 0.849524 Site_Lymph_Node 414.0 86.0 73.0 487.0 0.828000 0.850103 0.838906 Invasion 21.0 8.0 28.0 49.0 0.724138 0.428571 0.538462 Response_To_Treatment 249.0 64.0 112.0 361.0 0.795527 0.689751 0.738872 Smoking_Status 18.0 10.0 7.0 25.0 0.642857 0.720000 0.679245 Tumor_Size 623.0 69.0 37.0 660.0 0.900289 0.943939 0.921598 Age 416.0 1.0 23.0 439.0 0.997602 0.947608 0.971963 Biomarker_Result 539.0 122.0 202.0 741.0 0.815431 0.727395 0.768902 Unspecific_Therapy 57.0 30.0 52.0 109.0 0.655172 0.522936 0.581633 Chemotherapy 415.0 29.0 25.0 440.0 0.934685 0.943182 0.938914 Targeted_Therapy 104.0 33.0 10.0 114.0 0.759124 0.912281 0.828685 Radiotherapy 104.0 10.0 26.0 130.0 0.912281 0.800000 0.852459 Performance_Status 61.0 6.0 35.0 96.0 0.910448 0.635417 0.748466 Pathology_Test 352.0 68.0 118.0 470.0 0.838095 0.748936 0.791011 Cancer_Surgery 291.0 41.0 38.0 329.0 0.876506 0.884498 0.880484 Line_Of_Therapy 64.0 5.0 11.0 75.0 0.927536 0.853333 0.888889 Pathology_Result 154.0 156.0 82.0 236.0 0.496774 0.652542 0.564103 Hormonal_Therapy 62.0 3.0 14.0 76.0 0.953846 0.815789 0.879433 Biomarker 734.0 236.0 103.0 837.0 0.756701 0.876941 0.812396 Immunotherapy 24.0 7.0 19.0 43.0 0.774194 0.558140 0.648649 Cycle_Day 58.0 21.0 10.0 68.0 0.734177 0.852941 0.789116 Frequency 192.0 17.0 53.0 245.0 0.918660 0.783673 0.845815 Route 26.0 3.0 41.0 67.0 0.896552 0.388060 0.541667 Duration 230.0 88.0 67.0 297.0 0.723270 0.774411 0.747967 Metastasis 171.0 2.0 22.0 193.0 0.988439 0.886010 0.934426 Cancer_Dx 568.0 59.0 59.0 627.0 0.905901 0.905901 0.905901 Grade 15.0 9.0 73.0 88.0 0.625000 0.170455 0.267857 Date 385.0 21.0 12.0 397.0 0.948276 0.969773 0.958904 Relative_Date 348.0 85.0 91.0 439.0 0.803695 0.792711 0.798165 Race_Ethnicity 21.0 1.0 3.0 24.0 0.954545 0.875000 0.913043 Gender 619.0 12.0 10.0 629.0 0.980983 0.984102 0.982540 Dosage 512.0 45.0 107.0 619.0 0.919210 0.827141 0.870748 Oncogene 283.0 23.0 120.0 403.0 0.924837 0.702233 0.798307 Radiation_Dose 53.0 13.0 3.0 56.0 0.803030 0.946429 0.868852 Macro-average - - - - - - 0.796909 Micro-average - - - - - - 0.835757 ``` --- layout: model title: Sentence Entity Resolver for CPT codes (procedures and measurements) - Augmented author: John Snow Labs name: sbiobertresolve_cpt_procedures_measurements_augmented date: 2021-07-02 tags: [licensed, en, entity_resolution, clinical] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps medical entities to CPT codes using Sentence Bert Embeddings. The corpus of this model has been extented to measurements, and this model is capable of mapping both procedures and measurement concepts/entities to CPT codes. Measurement codes are helpful in codifying medical entities related to tests and their results. ## Predicted Entities CPT codes and their descriptions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_measurements_augmented_en_3.1.0_3.0_1625257370771.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_measurements_augmented_en_3.1.0_3.0_1625257370771.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_cpt_procedures_measurements_augmented``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_jsl``` as NER model. ```Procedure``` set in ```.setWhiteList()```. ```sbiobertresolve_cpt_procedures_measurements_augmented``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_measurements_clinical``` as NER model. ```Measurements``` set in ```.setWhiteList()```. Merge ner_jsl and ner_measurements_clinical model chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") jsl_sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli','en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") cpt_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_cpt_procedures_measurements_augmented", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("cpt_code") cpt_pipelineModel = PipelineModel( stages = [ documentAssembler, jsl_sbert_embedder, cpt_resolver]) cpt_lp = LightPipeline(cpt_pipelineModel) result = cpt_lp.fullAnnotate(['calcium score', 'heart surgery']) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val cpt_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_cpt_procedures_measurements_augmented", "en", "clinical/models) .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("cpt_code") val cpt_pipelineModel= new PipelineModel().setStages(Array(document_assembler, sbert_embedder, cpt_resolver)) val cpt_lp = LightPipeline(cpt_pipelineModel) val result = cpt_lp.fullAnnotate(['calcium score', 'heart surgery']) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.cpt.procedures_measurements").predict("""calcium score""") ```

## Results ```bash | | chunks | code | resolutions | |---:|:--------------|:----- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | calcium score | 82310 | Calcium measurement [Calcium; total] | | 1 | heart surgery | 33257 | Cardiac surgery procedure [Operative tissue ablation and reconstruction of atria, performed at the time of other cardiac procedure(s), limited (eg, modified maze procedure) (List separately in addition to code for primary procedure)] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_cpt_procedures_measurements_augmented| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sbert_embeddings]| |Output Labels:|[cpt_code]| |Language:|en| |Case sensitive:|true| ## Data Source Trained on Current Procedural Terminology dataset with `sbiobert_base_cased_mli` sentence embeddings. --- layout: model title: RxNorm to UMLS Code Mapping author: John Snow Labs name: rxnorm_umls_mapping date: 2021-07-01 tags: [rxnorm, umls, en, licensed, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps RxNorm codes to UMLS codes without using any text data. You’ll just feed white space-delimited RxNorm codes and it will return the corresponding UMLS codes as a list. If there is no mapping, the original code is returned with no mapping. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_CODE_MAPPING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_3.1.0_2.4_1625126295049.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_3.1.0_2.4_1625126295049.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline( "rxnorm_umls_mapping","en","clinical/models") pipeline.annotate("1161611 315677 343663") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline( "rxnorm_umls_mapping","en","clinical/models") val result = pipeline.annotate("1161611 315677 343663") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.umls").predict("""1161611 315677 343663""") ```

## Results ```bash {'rxnorm': ['1161611', '315677', '343663'], 'umls': ['C3215948', 'C0984912', 'C1146501']} Note: | RxNorm | Details | | ---------- | ------------------------:| | 1161611 | metformin Pill | | 315677 | cimetidine 100 mg | | 343663 | insulin lispro 50 UNT/ML | | UMLS | Details | | ---------- | ------------------------:| | C3215948 | metformin pill | | C0984912 | cimetidine 100 mg | | C1146501 | insulin lispro 50 unt/ml | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: English Bert Embeddings (from jackaduma) author: John Snow Labs name: bert_embeddings_SecBERT date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `SecBERT` is a English model orginally trained by `jackaduma`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_SecBERT_en_3.4.2_3.0_1649671929316.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_SecBERT_en_3.4.2_3.0_1649671929316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_SecBERT| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|313.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/jackaduma/SecBERT - https://github.com/jackaduma/SecBERT/ - https://github.com/kbandla/APTnotes - https://stucco.github.io/data/ - https://ebiquity.umbc.edu/_file_directory_/papers/943.pdf - https://competitions.codalab.org/competitions/17262 - https://github.com/allenai/scibert - https://github.com/jackaduma/SecBERT - https://github.com/jackaduma/SecBERT --- layout: model title: English image_classifier_vit_base_cats_vs_dogs ViTForImageClassification from akahana author: John Snow Labs name: image_classifier_vit_base_cats_vs_dogs date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_cats_vs_dogs` is a English model originally trained by akahana. ## Predicted Entities `cat`, `dog` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_cats_vs_dogs_en_4.1.0_3.0_1660171931492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_cats_vs_dogs_en_4.1.0_3.0_1660171931492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_cats_vs_dogs", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_cats_vs_dogs", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_cats_vs_dogs| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from Arabic to Hebrew author: John Snow Labs name: opus_mt_ar_he date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, he, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: he {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_he_xx_3.1.0_2.4_1622552020327.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_he_xx_3.1.0_2.4_1622552020327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_he| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: French CamemBert Embeddings (from Katster) author: John Snow Labs name: camembert_embeddings_Katster_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Katster`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Katster_generic_model_fr_3.4.4_3.0_1653986534274.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Katster_generic_model_fr_3.4.4_3.0_1653986534274.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Katster_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Katster/dummy-model --- layout: model title: Multilingual T5ForConditionalGeneration Cased model (from Lucapro) author: John Snow Labs name: t5_test_model date: 2023-01-31 tags: [en, ro, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `test-model` is a Multilingual model originally trained by `Lucapro`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_test_model_xx_4.3.0_3.0_1675157312429.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_test_model_xx_4.3.0_3.0_1675157312429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_test_model","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_test_model","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_test_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|260.2 MB| ## References - https://huggingface.co/Lucapro/test-model - https://paperswithcode.com/sota?task=Translation&dataset=wmt16+ro-en --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el8_dl1 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el8-dl1` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_dl1_en_4.3.0_3.0_1675120557059.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_dl1_en_4.3.0_3.0_1675120557059.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el8_dl1","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el8_dl1","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el8_dl1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|119.7 MB| ## References - https://huggingface.co/google/t5-efficient-small-el8-dl1 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Spam Classifier author: John Snow Labs name: classifierdl_use_spam date: 2021-01-09 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, en] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Automatically identify messages as being regular messages or Spam. ## Predicted Entities `spam`, `ham` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_spam_en_2.7.1_2.4_1610187019592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_spam_en_2.7.1_2.4_1610187019592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------------------------------------------------------------------------+------------+ |document |class | +------------------------------------------------------------------------------------------------+------------+ |Congratulations! You've won a $1,000 Walmart gift card. Go to http://bit.ly/1234 to claim now. | spam | +------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_spam| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|tfhub_use| ## Data Source This model is trained on UCI spam dataset. https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip ## Benchmarking ```bash precision recall f1-score support ham 0.99 0.99 0.99 966 spam 0.95 0.95 0.95 149 accuracy 0.99 1115 macro avg 0.97 0.97 0.97 1115 weighted avg 0.99 0.99 0.99 1115 ``` --- layout: model title: English image_classifier_vit_Teeth_A ViTForImageClassification from steven123 author: John Snow Labs name: image_classifier_vit_Teeth_A date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Teeth_A` is a English model originally trained by steven123. ## Predicted Entities `Good Teeth`, `Missing Teeth`, `Rotten Teeth` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Teeth_A_en_4.1.0_3.0_1660170593463.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Teeth_A_en_4.1.0_3.0_1660170593463.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Teeth_A", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Teeth_A", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Teeth_A| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect Normalized Genes and Human Phenotypes author: John Snow Labs name: ner_human_phenotype_go_clinical_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_go_clinical](https://nlp.johnsnowlabs.com/2021/03/31/ner_human_phenotype_go_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_clinical_pipeline_en_4.3.0_3.2_1678874608526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_clinical_pipeline_en_4.3.0_3.2_1678874608526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_human_phenotype_go_clinical_pipeline", "en", "clinical/models") text = '''Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_human_phenotype_go_clinical_pipeline", "en", "clinical/models") val text = "Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phenotype_clinical.pipeline").predict("""Another disease that shares two of the tumor components of CT, namely GIST and tricarboxylic acid cycle is the Carney-Stratakis syndrome (CSS) or dyad.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------------------|--------:|------:|:------------|-------------:| | 0 | tumor | 39 | 43 | HP | 0.9996 | | 1 | tricarboxylic acid cycle | 79 | 102 | GO | 0.994633 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_go_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal Board Of Directors Clause Binary Classifier author: John Snow Labs name: legclf_board_of_directors_clause date: 2023-01-29 tags: [en, legal, classification, board, directors, clauses, board_of_directors, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `board-of-directors` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `board-of-directors`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_board_of_directors_clause_en_1.0.0_3.0_1675004773054.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_board_of_directors_clause_en_1.0.0_3.0_1675004773054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_board_of_directors_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[board-of-directors]| |[other]| |[other]| |[board-of-directors]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_board_of_directors_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support board-of-directors 1.00 0.96 0.98 24 other 0.97 1.00 0.99 38 accuracy - - 0.98 62 macro-avg 0.99 0.98 0.98 62 weighted-avg 0.98 0.98 0.98 62 ``` --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_modelo_v1b3 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-modelo-robertav1b3` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo_v1b3_es_4.3.0_3.0_1674218384530.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo_v1b3_es_4.3.0_3.0_1674218384530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo_v1b3","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo_v1b3","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_modelo_v1b3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-modelo-robertav1b3 --- layout: model title: English asr_vakyansh_wav2vec2_maithili_maim_50 TFWav2Vec2ForCTC from Harveenchadha author: John Snow Labs name: pipeline_asr_vakyansh_wav2vec2_maithili_maim_50 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_vakyansh_wav2vec2_maithili_maim_50` is a English model originally trained by Harveenchadha. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_vakyansh_wav2vec2_maithili_maim_50_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_vakyansh_wav2vec2_maithili_maim_50_en_4.2.0_3.0_1664117308568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_vakyansh_wav2vec2_maithili_maim_50_en_4.2.0_3.0_1664117308568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_vakyansh_wav2vec2_maithili_maim_50', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_vakyansh_wav2vec2_maithili_maim_50", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_vakyansh_wav2vec2_maithili_maim_50| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect PHI for Deidentification in Romanian (Word2Vec) author: John Snow Labs name: ner_deid_subentity date: 2022-06-27 tags: [ner, deidentification, word2vec, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Romanian) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 17 entities. This NER model is trained with a combination of custom datasets with several data augmentation mechanisms. ## Predicted Entities `AGE`, `CITY`, `COUNTRY`, `DATE`, `DOCTOR`, `EMAIL`, `FAX`, `HOSPITAL`, `IDNUM`, `LOCATION-OTHER`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `ZIP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ro_4.0.0_3.0_1656316441636.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_ro_4.0.0_3.0_1656316441636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------+---------+ |chunk |ner_label| +----------------------------+---------+ |Spitalul Pentru Ochi de Deal|HOSPITAL | |Drumul Oprea Nr. |STREET | |Vaslui |CITY | |737405 |ZIP | |+40(235)413773 |PHONE | |25 May 2022 |DATE | |BUREAN MARIA |PATIENT | |77 |AGE | |Agota Evelyn |DOCTOR | |2450502264401 |IDNUM | +----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|15.1 MB| ## References - Custom John Snow Labs datasets - Data augmentation techniques ## Benchmarking ```bash label precision recall f1-score support AGE 0.96 0.99 0.97 1235 CITY 0.97 0.95 0.96 307 COUNTRY 0.92 0.74 0.82 115 DATE 0.94 0.89 0.91 5006 DOCTOR 0.96 0.96 0.96 2064 EMAIL 1.00 1.00 1.00 8 FAX 1.00 0.95 0.97 56 HOSPITAL 0.78 0.83 0.80 919 IDNUM 0.98 1.00 0.99 239 LOCATION-OTHER 1.00 0.85 0.92 13 MEDICALRECORD 1.00 1.00 1.00 455 ORGANIZATION 0.34 0.41 0.37 82 PATIENT 0.85 0.90 0.87 954 PHONE 0.97 0.98 0.98 315 PROFESSION 0.87 0.80 0.83 173 STREET 0.99 0.99 0.99 174 ZIP 0.99 0.97 0.98 140 micro-avg 0.92 0.91 0.92 12255 macro-avg 0.91 0.89 0.90 12255 weighted-avg 0.93 0.91 0.92 12255 ``` --- layout: model title: Detect Drugs and Proteins author: John Snow Labs name: ner_drugprot_clinical date: 2021-12-20 tags: [ner, clinical, drugprot, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects chemical compounds/drugs and genes/proteins in medical text and research articles. Chemical compounds/drugs are labeled as `CHEMICAL`, genes/proteins are labeled as `GENE` and entity mentions of type `GENE` and of type `CHEMICAL` that overlap such as enzymes and small peptides are labeled as `GENE_AND_CHEMICAL`. ## Predicted Entities `GENE`, `CHEMICAL`, `GENE_AND_CHEMICAL` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DRUG_PROT/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugprot_clinical_en_3.3.3_3.0_1639989110299.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugprot_clinical_en_3.3.3_3.0_1639989110299.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |clenbuterol |CHEMICAL | |beta 2-adrenoceptor |GENE | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugprot_clinical| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|14.7 MB| |Dependencies:|embeddings_clinical| ## Data Source This model was trained on the [DrugProt corpus](https://zenodo.org/record/5119892). ## Benchmarking ```bash label tp fp fn total precision recall f1 GENE_AND_CHEMICAL 786.0 171.0 143.0 929.0 0.8213 0.8461 0.8335 CHEMICAL 8228.0 779.0 575.0 8803.0 0.9135 0.9347 0.924 GENE 7176.0 822.0 652.0 7828.0 0.8972 0.9167 0.9069 macro - - - - - - 0.88811683 micro - - - - - - 0.91156048 ``` --- layout: model title: English BertForQuestionAnswering model (from AnonymousSub) author: John Snow Labs name: bert_qa_news_pretrain_bert_FT_newsqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `news_pretrain_bert_FT_newsqa` is a English model orginally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_news_pretrain_bert_FT_newsqa_en_4.0.0_3.0_1654188933131.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_news_pretrain_bert_FT_newsqa_en_4.0.0_3.0_1654188933131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_news_pretrain_bert_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_news_pretrain_bert_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.bert.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_news_pretrain_bert_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/news_pretrain_bert_FT_newsqa --- layout: model title: Stopwords Remover for Tigrinya language (182 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ti, open_source] task: Stop Words Removal language: ti edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ti_3.4.1_3.0_1646673030296.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ti_3.4.1_3.0_1646673030296.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+ |result | +---------------------------+ |[ይቕሬታ፣, ሓደ, መደብ, ገይረ, ኣሎኹ።]| +---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ti| |Size:|2.1 KB| --- layout: model title: Modern Greek (1453-) BertForQuestionAnswering model (from Danastos) author: John Snow Labs name: bert_qa_triviaqa_bert_el_Danastos date: 2022-06-03 tags: [open_source, question_answering, bert] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `triviaqa_bert_el` is a Modern Greek (1453-) model orginally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_triviaqa_bert_el_Danastos_el_4.0.0_3.0_1654250055165.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_triviaqa_bert_el_Danastos_el_4.0.0_3.0_1654250055165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_triviaqa_bert_el_Danastos","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_triviaqa_bert_el_Danastos","el") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("el.answer_question.trivia.bert.by_Danastos").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_triviaqa_bert_el_Danastos| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|el| |Size:|421.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/triviaqa_bert_el --- layout: model title: Detect Clinical Entities (ner_eu_clinical_case - eu) author: John Snow Labs name: ner_eu_clinical_case date: 2023-02-02 tags: [eu, clinical, licensed, ner] task: Named Entity Recognition language: eu edition: Healthcare NLP 4.2.8 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition (NER) deep learning model for extracting clinical entities from Basque texts. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Predicted Entities `clinical_event`, `bodypart`, `clinical_condition`, `units_measurements`, `patient`, `date_time` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_eu_4.2.8_3.0_1675359410041.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_eu_clinical_case_eu_4.2.8_3.0_1675359410041.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","eu")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_eu_clinical_case', "eu", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""3 urteko mutiko bat nahasmendu autistarekin unibertsitateko ospitaleko A pediatriako ospitalean. Ez du autismoaren espektroaren nahaste edo gaixotasun familiaren aurrekaririk. Mutilari komunikazio-nahaste larria diagnostikatu zioten, elkarrekintza sozialeko zailtasunak eta prozesamendu sentsorial atzeratua. Odol-analisiak normalak izan ziren (tiroidearen hormona estimulatzailea (TSH), hemoglobina, batez besteko bolumen corpuskularra (MCV) eta ferritina). Goiko endoskopiak mukosaren azpiko tumore bat ere erakutsi zuen, urdail-irteeren guztizko oztopoa eragiten zuena. Estroma gastrointestinalaren tumore baten susmoa ikusita, distaleko gastrektomia egin zen. Azterketa histopatologikoak agerian utzi zuen mukosaren azpiko zelulen ugaltzea."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","eu") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_eu_clinical_case", "eu", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""3 urteko mutiko bat nahasmendu autistarekin unibertsitateko ospitaleko A pediatriako ospitalean. Ez du autismoaren espektroaren nahaste edo gaixotasun familiaren aurrekaririk. Mutilari komunikazio-nahaste larria diagnostikatu zioten, elkarrekintza sozialeko zailtasunak eta prozesamendu sentsorial atzeratua. Odol-analisiak normalak izan ziren (tiroidearen hormona estimulatzailea (TSH), hemoglobina, batez besteko bolumen corpuskularra (MCV) eta ferritina). Goiko endoskopiak mukosaren azpiko tumore bat ere erakutsi zuen, urdail-irteeren guztizko oztopoa eragiten zuena. Estroma gastrointestinalaren tumore baten susmoa ikusita, distaleko gastrektomia egin zen. Azterketa histopatologikoak agerian utzi zuen mukosaren azpiko zelulen ugaltzea.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------------+------------------+ |chunk |ner_label | +----------------------------+------------------+ |3 urteko mutiko bat |patient | |nahasmendu |clinical_event | |autismoaren espektroaren |clinical_condition| |nahaste |clinical_event | |gaixotasun |clinical_event | |familiaren |patient | |aurrekaririk |clinical_event | |Mutilari |patient | |komunikazio-nahaste |clinical_event | |diagnostikatu |clinical_event | |elkarrekintza |clinical_event | |zailtasunak |clinical_event | |prozesamendu sentsorial |clinical_event | |Odol-analisiak |clinical_event | |normalak |units_measurements| |tiroidearen |bodypart | |hormona estimulatzailea |clinical_event | |TSH |clinical_event | |hemoglobina |clinical_event | |bolumen |clinical_event | |MCV |clinical_event | |ferritina |clinical_event | |Goiko |bodypart | |endoskopiak |clinical_event | |mukosaren azpiko |bodypart | |tumore |clinical_event | |erakutsi |clinical_event | |oztopoa |clinical_event | |Estroma gastrointestinalaren|clinical_event | |tumore |clinical_event | |ikusita |clinical_event | |distaleko |bodypart | |gastrektomia |clinical_event | |Azterketa |clinical_event | |agerian |clinical_event | |utzi |clinical_event | |mukosaren azpiko zelulen |bodypart | |ugaltzea |clinical_event | +----------------------------+------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_case| |Compatibility:|Healthcare NLP 4.2.8+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|eu| |Size:|896.1 KB| ## References The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Sample text from the training dataset 3 urteko mutiko bat nahasmendu autistarekin unibertsitateko ospitaleko A pediatriako ospitalean. Ez du autismoaren espektroaren nahaste edo gaixotasun familiaren aurrekaririk. Mutilari komunikazio-nahaste larria diagnostikatu zioten, elkarrekintza sozialeko zailtasunak eta prozesamendu sentsorial atzeratua. Odol-analisiak normalak izan ziren (tiroidearen hormona estimulatzailea (TSH), hemoglobina, batez besteko bolumen corpuskularra (MCV) eta ferritina). Goiko endoskopiak mukosaren azpiko tumore bat ere erakutsi zuen, urdail-irteeren guztizko oztopoa eragiten zuena. Estroma gastrointestinalaren tumore baten susmoa ikusita, distaleko gastrektomia egin zen. Azterketa histopatologikoak agerian utzi zuen mukosaren azpiko zelulen ugaltzea. ## Benchmarking ```bash label tp fp fn total precision recall f1 date_time 103.0 13.0 26.0 129.0 0.8879 0.7984 0.8408 units_measurements 257.0 37.0 9.0 266.0 0.8741 0.9662 0.9179 clinical_condition 20.0 22.0 33.0 53.0 0.4782 0.3774 0.4211 patient 69.0 3.0 8.0 77.0 0.9583 0.8961 0.9262 clinical_event 712.0 121.0 95.0 807.0 0.8547 0.8823 0.8683 bodypart 182.0 33.0 15.0 197.0 0.8465 0.9239 0.8835 macro - - - - - - 0.8096 micro - - - - - - 0.8640 ``` --- layout: model title: English ElectraForQuestionAnswering model (from bdickson) Version-1 author: John Snow Labs name: electra_qa_small_discriminator_finetuned_squad_1 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-small-discriminator-finetuned-squad` is a English model originally trained by `bdickson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_small_discriminator_finetuned_squad_1_en_4.0.0_3.0_1655921242790.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_small_discriminator_finetuned_squad_1_en_4.0.0_3.0_1655921242790.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_small_discriminator_finetuned_squad_1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_small_discriminator_finetuned_squad_1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.small.by_bdickson").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_small_discriminator_finetuned_squad_1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|51.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bdickson/electra-small-discriminator-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Russian author: John Snow Labs name: opus_mt_en_ru date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ru, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ru` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ru_xx_2.7.0_2.4_1609164756740.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ru_xx_2.7.0_2.4_1609164756740.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ru| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for RxNorm (sbluebert_base_uncased_mli embeddings) author: John Snow Labs name: sbluebertresolve_rxnorm_augmented_uncased date: 2021-12-28 tags: [en, clinical, entity_resolution, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbluebert_base_uncased_mli` Sentence Bert Embeddings. It is trained on the augmented version of the dataset which is used in previous RxNorm resolver models. Additionally, this model returns concept classes of the drugs in all_k_aux_labels column. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbluebertresolve_rxnorm_augmented_uncased_en_3.3.4_2.4_1640698320751.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbluebertresolve_rxnorm_augmented_uncased_en_3.3.4_2.4_1640698320751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings.pretrained('sbluebert_base_uncased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbluebertresolve_rxnorm_augmented_uncased", "en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver]) light_model = LightPipeline(rxnorm_pipelineModel) result = light_model.fullAnnotate(["Coumadin 5 mg", "aspirin", "Neurontin 300", "avandia 4 mg"]) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbluebert_base_uncased_mli", "en", "clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbluebertresolve_rxnorm_augmented_uncased", "en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_pipelineModel = new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val light_model = LightPipeline(rxnorm_pipelineModel) val result = light_model.fullAnnotate(Array("Coumadin 5 mg", "aspirin", "avandia 4 mg")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.augmented_uncased").predict("""Coumadin 5 mg""") ```

## Results ```bash | | RxNormCode | Resolution | all_k_results | all_k_distances | all_k_cosine_distances | all_k_resolutions | all_k_aux_labels | |---:|-------------:|:-------------------------------------------|:----------------------------------|:----------------------------------|:----------------------------------|:----------------------------------------------------------------|:----------------------------------| | 0 | 855333 | warfarin sodium 5 MG [Coumadin] | 855333:::432467:::438740:::855... | 0.0000:::1.6841:::1.6841:::3.2... | 0.0000:::0.0062:::0.0062:::0.0... | warfarin sodium 5 MG [Coumadin]:::coumarin 5 MG Oral Tablet:... | Branded Drug Comp:::Clinical D... | | 1 | 1537020 | aspirin Effervescent Oral Tablet | 1537020:::1191:::405403:::1001... | 0.0000:::0.0000:::6.0493:::6.4... | 0.0000:::0.0000:::0.0797:::0.0... | aspirin Effervescent Oral Tablet:::aspirin:::YSP Aspirin:::E... | Clinical Drug Form:::Ingredien... | | 2 | 105029 | gabapentin 300 MG Oral Capsule [Neurontin] | 105029:::1098609:::207088:::20... | 3.1683:::6.0071:::6.2050:::6.2... | 0.0227:::0.0815:::0.0862:::0.0... | gabapentin 300 MG Oral Capsule [Neurontin]:::lamotrigine 300... | Branded Drug:::Branded Drug Co... | | 3 | 261242 | rosiglitazone 4 MG Oral Tablet [Avandia] | 261242:::847706:::577784:::212... | 0.0000:::6.8783:::6.9828:::7.4... | 0.0000:::0.1135:::0.1183:::0.1... | rosiglitazone 4 MG Oral Tablet [Avandia]:::glimepiride 4 MG ... | Branded Drug:::Branded Drug Co... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbluebertresolve_rxnorm_augmented_uncased| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|978.4 MB| |Case sensitive:|false| --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-16-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0_en_4.0.0_3.0_1655731514790.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0_en_4.0.0_3.0_1655731514790.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_16_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|415.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-16-finetuned-squad-seed-0 --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_nl16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-nl16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl16_en_4.3.0_3.0_1675117543768.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_nl16_en_4.3.0_3.0_1675117543768.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_nl16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_nl16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_nl16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|1.0 GB| ## References - https://huggingface.co/google/t5-efficient-large-nl16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Japanese Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_bert_large_japanese date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-japanese` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_japanese_ja_3.4.2_3.0_1649674459280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_japanese_ja_3.4.2_3.0_1649674459280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_japanese| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-large-japanese - https://github.com/google-research/bert - https://pypi.org/project/unidic-lite/ - https://github.com/cl-tohoku/bert-japanese/tree/v2.0 - https://taku910.github.io/mecab/ - https://github.com/neologd/mecab-ipadic-neologd - https://github.com/polm/fugashi - https://github.com/polm/unidic-lite - https://www.tensorflow.org/tfrc/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: English asr_wav2vec2_base_timit_ali_hasan_colab_EX2 TFWav2Vec2ForCTC from ali221000262 author: John Snow Labs name: asr_wav2vec2_base_timit_ali_hasan_colab_EX2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_ali_hasan_colab_EX2` is a English model originally trained by ali221000262. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_ali_hasan_colab_EX2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_ali_hasan_colab_EX2_en_4.2.0_3.0_1664038518590.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_ali_hasan_colab_EX2_en_4.2.0_3.0_1664038518590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_ali_hasan_colab_EX2", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_ali_hasan_colab_EX2", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_ali_hasan_colab_EX2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: Ganda asr_wav2vec2_luganda_by_birgermoell TFWav2Vec2ForCTC from birgermoell author: John Snow Labs name: asr_wav2vec2_luganda_by_birgermoell date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_luganda_by_birgermoell` is a Ganda model originally trained by birgermoell. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_luganda_by_birgermoell_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_birgermoell_lg_4.2.0_3.0_1664037165332.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_birgermoell_lg_4.2.0_3.0_1664037165332.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_luganda_by_birgermoell", "lg")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_luganda_by_birgermoell", "lg") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_luganda_by_birgermoell| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lg| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: bert_qa_bert_multi_uncased_finetuned_chaii date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-uncased-finetuned-chaii` is a English model orginally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_uncased_finetuned_chaii_en_4.0.0_3.0_1654184608840.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_uncased_finetuned_chaii_en_4.0.0_3.0_1654184608840.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_multi_uncased_finetuned_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_multi_uncased_finetuned_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.bert.uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_multi_uncased_finetuned_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|626.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/bert-multi-uncased-finetuned-chaii --- layout: model title: English BertForQuestionAnswering model (from Neulvo) author: John Snow Labs name: bert_qa_Neulvo_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `Neulvo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Neulvo_bert_finetuned_squad_en_4.0.0_3.0_1654535543683.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Neulvo_bert_finetuned_squad_en_4.0.0_3.0_1654535543683.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Neulvo_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Neulvo_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_Neulvo").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Neulvo_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Neulvo/bert-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Welsh author: John Snow Labs name: opus_mt_en_cy date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, cy, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `cy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_cy_xx_2.7.0_2.4_1609170126158.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_cy_xx_2.7.0_2.4_1609170126158.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_cy| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_cetuc_sid_voxforge_mls_1 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_cetuc_sid_voxforge_mls_1` is a English model originally trained by joaoalvarenga. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1_en_4.2.0_3.0_1664023274170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1_en_4.2.0_3.0_1664023274170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_cetuc_sid_voxforge_mls_1| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering Cased model (from Slavka) author: John Snow Labs name: bert_qa_xdzadi00_based_v4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xdzadi00_bert-based-v4` is a English model originally trained by `Slavka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_xdzadi00_based_v4_en_4.0.0_3.0_1657193206660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_xdzadi00_based_v4_en_4.0.0_3.0_1657193206660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_xdzadi00_based_v4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_xdzadi00_based_v4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_xdzadi00_based_v4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Slavka/xdzadi00_bert-based-v4 --- layout: model title: Legal Revolving Credit Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_revolving_credit_agreement_bert date: 2022-12-06 tags: [en, legal, classification, agreement, revolving, credit, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_revolving_credit_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `revolving-credit-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `revolving-credit-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_revolving_credit_agreement_bert_en_1.0.0_3.0_1670349861196.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_revolving_credit_agreement_bert_en_1.0.0_3.0_1670349861196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_revolving_credit_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[revolving-credit-agreement]| |[other]| |[other]| |[revolving-credit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_revolving_credit_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.94 0.94 35 revolving-credit-agreement 0.93 0.93 0.93 29 accuracy - - 0.94 64 macro-avg 0.94 0.94 0.94 64 weighted-avg 0.94 0.94 0.94 64 ``` --- layout: model title: NER Pipeline for Clinical Department - Voice of the Patient author: John Snow Labs name: ner_vop_clinical_dept_pipeline date: 2023-06-10 tags: [licensed, ner, en, vop, pipeline] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline extracts mentions of clinical departments and medical devices from health-related text in colloquial language. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_pipeline_en_4.4.3_3.0_1686406177343.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_clinical_dept_pipeline_en_4.4.3_3.0_1686406177343.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_vop_clinical_dept_pipeline", "en", "clinical/models") pipeline.annotate(" My little brother is having surgery tomorrow in the orthopedic department. He is getting a titanium plate put in his leg to help it heal faster. Wishing him a speedy recovery! ") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_vop_clinical_dept_pipeline", "en", "clinical/models") val result = pipeline.annotate(" My little brother is having surgery tomorrow in the orthopedic department. He is getting a titanium plate put in his leg to help it heal faster. Wishing him a speedy recovery! ") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_tinybert_6l_768d_squad2_large_teach","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_tinybert_6l_768d_squad2_large_teach","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_tinybert_6l_768d_squad2_large_teach| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|249.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MichelBartels/tinybert-6l-768d-squad2-large-teacher --- layout: model title: Multilingual BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_zh_hi_dev date: 2022-07-07 tags: [xx, open_source, bert, question_answering] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-zh-hi` is a Multilingual model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_zh_hi_dev_xx_4.0.0_3.0_1657190323203.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_zh_hi_dev_xx_4.0.0_3.0_1657190323203.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_zh_hi_dev","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_zh_hi_dev","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_zh_hi_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-zh-hi --- layout: model title: English T5ForConditionalGeneration Mini Cased model (from google) author: John Snow Labs name: t5_efficient_mini_nl8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-mini-nl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_mini_nl8_en_4.3.0_3.0_1675118402070.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_mini_nl8_en_4.3.0_3.0_1675118402070.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_mini_nl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_mini_nl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_mini_nl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|120.1 MB| ## References - https://huggingface.co/google/t5-efficient-mini-nl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Sublicense Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_sublicense_agreement_bert date: 2023-01-29 tags: [en, legal, classification, sublicense, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_sublicense_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `sublicense-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `sublicense-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sublicense_agreement_bert_en_1.0.0_3.0_1674990525969.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sublicense_agreement_bert_en_1.0.0_3.0_1674990525969.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_sublicense_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[sublicense-agreement]| |[other]| |[other]| |[sublicense-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sublicense_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.93 0.99 0.96 99 sublicense-agreement 0.97 0.83 0.89 41 accuracy - - 0.94 140 macro-avg 0.95 0.91 0.93 140 weighted-avg 0.94 0.94 0.94 140 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Sivakumar) author: John Snow Labs name: distilbert_qa_sivakumar_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Sivakumar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sivakumar_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769288959.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sivakumar_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769288959.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sivakumar_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sivakumar_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sivakumar_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Sivakumar/distilbert-base-uncased-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from Finno-Ugrian Languages to English author: John Snow Labs name: opus_mt_fiu_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, fiu, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `fiu` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_fiu_en_xx_2.7.0_2.4_1609164957344.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_fiu_en_xx_2.7.0_2.4_1609164957344.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_fiu_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Оcr small for printed text author: John Snow Labs name: ocr_small_printed date: 2023-01-10 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 3.3.3 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Ocr small model for recognise printed text based on TrOcr architecture. The TrOCR model was proposed in TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. TrOCR consists of an image Transformer encoder and an autoregressive text Transformer decoder to perform optical character recognition (OCR). The abstract from the paper is the following: Text recognition is a long-standing research problem for document digitalization. Existing approaches for text recognition are usually built based on CNN for image understanding and RNN for char-level text generation. In addition, another language model is usually needed to improve the overall accuracy as a post-processing step. In this paper, we propose an end-to-end text recognition approach with pre-trained image Transformer and text Transformer models, namely TrOCR, which leverages the Transformer architecture for both image understanding and wordpiece-level text generation. The TrOCR model is simple but effective, and can be pre-trained with large-scale synthetic data and fine-tuned with human-labeled datasets. Experiments show that the TrOCR model outperforms the current state-of-the-art models on both printed and handwritten text recognition tasks. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/RECOGNIZE_PRINTED/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/tutorials/Certification_Trainings/1.3.Trasformer_based_Text_Recognition.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_printed_en_3.3.3_2.4_1645007455031.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/ocr_small_printed_en_3.3.3_2.4_1645007455031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) text_detector = ImageTextDetectorV2 \ .pretrained("image_text_detector_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("text_regions") \ .setWithRefiner(False) \ .setSizeThreshold(15) \ .setLinkThreshold(0.3) draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("text_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.green) \ .setRotated(True) ocr = ImageToTextV2.pretrained("ocr_small_printed", "en", "clinical/ocr") \ .setInputCols(["image", "text_regions"]) \ .setOutputCol("hocr") \ .setOutputFormat(OcrOutputFormat.HOCR) \ .setGroupImages(False) pipeline = PipelineModel(stages=[ binary_to_image, text_detector, draw_regions, ocr ]) image_path = pkg_resources.resource_filename('sparkocr', 'resources/ocr/images/check.jpg') image_example_df = spark.read.format("binaryFile").load(image_path) result = pipeline.transform(image_example_df).cache() ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val text_detector = ImageTextDetectorV2.pretrained("image_text_detector_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("text_regions") .setWithRefiner(False) .setSizeThreshold(15) .setLinkThreshold(0.3) val draw_regions = ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("text_regions") .setOutputCol("image_with_regions") .setRectColor(Color.green) .setRotated(True) val ocr = ImageToTextV2.pretrained("ocr_small_printed", "en", "clinical/ocr") .setInputCols("image", "text_regions") .setOutputCol("hocr") .setOutputFormat(OcrOutputFormat.HOCR) .setGroupImages(False) val pipeline = new PipelineModel().setStages(Array( binary_to_image, text_detector, draw_regions, ocr)) val image_path = pkg_resources.resource_filename("sparkocr", "resources/ocr/images/check.jpg"") val image_example_df = spark.read.format("binaryFile").load(image_path) val result = pipeline.transform(image_example_df).cache() ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image2.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image2_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash STARBUCKS STORE #10208 11302 EUCLID AVENUE CLEVELAND OH (216) 229-0749 CHK 664290 12/07/2014 06:43 PM 1912003 DRAMER: 2 REG: 2 VT PEP MOCHA 4.95 SBUX CARD 4.95 XXXXXXXXXXXX3228 SUBTOTAL $4.95 TOTAL $4.95 CHANGE DUE $0.00 CHECK CLOSED 12/07/2014 06:43 PM SBUX CARD X3228 NEW BALANCE: 37.45 CARD IS REGISTERED ``` ## Model Information {:.table-model} |---|---| |Model Name:|ocr_small_printed| |Type:|ocr| |Compatibility:|Visual NLP 3.3.3+| |License:|Licensed| |Edition:|Official| |Language:|en| --- layout: model title: Chinese Lemmatizer author: John Snow Labs name: lemma date: 2020-05-04 20:11:00 +0800 task: Lemmatization language: zh edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, zh] supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_zh_2.5.0_2.4_1588611649140.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_zh_2.5.0_2.4_1588611649140.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "zh") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "zh") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""除了担任北方国王之外，约翰·斯诺（John Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。"""] lemma_df = nlu.load('zh.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=20, result='除了担任北方国王之外，约翰·斯诺（John', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=22, end=50, result='Snow）是一位英国医师，也是麻醉和医疗卫生发展的领导者。', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|zh| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Detect Living Species author: John Snow Labs name: bert_token_classifier_ner_living_species date: 2022-06-27 tags: [it, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: it edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Italian which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_it_3.5.3_3.0_1656318503132.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_it_3.5.3_3.0_1656318503132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "it", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, ner_model, ner_converter ]) data = spark.createDataFrame([["""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_living_species", "it", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("ner") .setCaseSensitive(True) .setMaxSentenceLength(512) val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, ner_model, ner_converter)) val data = Seq("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("it.classify.bert_token.ner_living_species").predict("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""") ```

## Results ```bash +----------------+-------+ |ner_chunk |label | +----------------+-------+ |donna |HUMAN | |personale |HUMAN | |madre |HUMAN | |fratello |HUMAN | |sorella |HUMAN | |virus epatotropi|SPECIES| |HBV |SPECIES| |HCV |SPECIES| |HIV |SPECIES| |paziente |HUMAN | +----------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|it| |Size:|410.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.78 0.96 0.86 2772 B-SPECIES 0.62 0.89 0.73 2866 I-HUMAN 0.44 0.63 0.52 101 I-SPECIES 0.57 0.83 0.67 1039 micro-avg 0.67 0.91 0.77 6778 macro-avg 0.60 0.83 0.70 6778 weighted-avg 0.68 0.91 0.77 6778 ``` --- layout: model title: Word2Vec Embeddings in Bangla (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, bn, open_source] task: Embeddings language: bn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_bn_3.4.1_3.0_1647286875201.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_bn_3.4.1_3.0_1647286875201.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_128d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|381.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-128-finetuned-squad-seed-4 --- layout: model title: English asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4 TFWav2Vec2ForCTC from chrisvinsen author: John Snow Labs name: pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4` is a English model originally trained by chrisvinsen. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_en_4.2.0_3.0_1664103715058.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4_en_4.2.0_3.0_1664103715058.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_4| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from Chinyanja to English author: John Snow Labs name: opus_mt_ny_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ny, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ny` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ny_en_xx_2.7.0_2.4_1609170818337.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ny_en_xx_2.7.0_2.4_1609170818337.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ny_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Taxation Document Classifier (EURLEX) author: John Snow Labs name: legclf_taxation_bert date: 2023-03-06 tags: [en, legal, classification, clauses, taxation, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_taxation_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Taxation or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Taxation`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_taxation_bert_en_1.0.0_3.0_1678111555574.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_taxation_bert_en_1.0.0_3.0_1678111555574.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_taxation_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Taxation]| |[Other]| |[Other]| |[Taxation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_taxation_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.86 0.94 0.90 96 Taxation 0.93 0.85 0.89 100 accuracy - - 0.89 196 macro-avg 0.90 0.89 0.89 196 weighted-avg 0.90 0.89 0.89 196 ``` --- layout: model title: Pipeline to Extract Biomarkers and their Results author: John Snow Labs name: ner_oncology_biomarker_pipeline date: 2023-03-09 tags: [licensed, clinical, en, ner, oncology, biomarker] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_biomarker](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_biomarker_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_pipeline_en_4.3.0_3.2_1678345026649.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_pipeline_en_4.3.0_3.2_1678345026649.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_biomarker_pipeline", "en", "clinical/models") text = '''The results of immunohistochemical examination showed that she tested negative for CK7, synaptophysin (Syn), chromogranin A (CgA), Muc5AC, human epidermal growth factor receptor-2 (HER2), and Muc6; positive for CK20, Muc1, Muc2, E-cadherin, and p53; the Ki-67 index was about 87%.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_biomarker_pipeline", "en", "clinical/models") val text = "The results of immunohistochemical examination showed that she tested negative for CK7, synaptophysin (Syn), chromogranin A (CgA), Muc5AC, human epidermal growth factor receptor-2 (HER2), and Muc6; positive for CK20, Muc1, Muc2, E-cadherin, and p53; the Ki-67 index was about 87%." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------------------------------|--------:|------:|:-----------------|-------------:| | 0 | negative | 70 | 77 | Biomarker_Result | 0.9984 | | 1 | CK7 | 83 | 85 | Biomarker | 1 | | 2 | synaptophysin | 88 | 100 | Biomarker | 0.9995 | | 3 | Syn | 103 | 105 | Biomarker | 0.9979 | | 4 | chromogranin A | 109 | 122 | Biomarker | 0.9692 | | 5 | CgA | 125 | 127 | Biomarker | 0.9994 | | 6 | Muc5AC | 131 | 136 | Biomarker | 0.9987 | | 7 | human epidermal growth factor receptor-2 | 139 | 178 | Biomarker | 0.99876 | | 8 | HER2 | 181 | 184 | Biomarker | 1 | | 9 | Muc6 | 192 | 195 | Biomarker | 0.9999 | | 10 | positive | 198 | 205 | Biomarker_Result | 0.9987 | | 11 | CK20 | 211 | 214 | Biomarker | 1 | | 12 | Muc1 | 217 | 220 | Biomarker | 0.9999 | | 13 | Muc2 | 223 | 226 | Biomarker | 0.9999 | | 14 | E-cadherin | 229 | 238 | Biomarker | 0.9999 | | 15 | p53 | 245 | 247 | Biomarker | 1 | | 16 | Ki-67 index | 254 | 264 | Biomarker | 0.99465 | | 17 | 87% | 276 | 278 | Biomarker_Result | 0.9814 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_biomarker_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering (from thatdramebaazguy) author: John Snow Labs name: roberta_qa_movie_roberta_MITmovie_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `movie-roberta-MITmovie-squad` is a English model originally trained by `thatdramebaazguy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_movie_roberta_MITmovie_squad_en_4.0.0_3.0_1655729031815.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_movie_roberta_MITmovie_squad_en_4.0.0_3.0_1655729031815.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_movie_roberta_MITmovie_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_movie_roberta_MITmovie_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.mitmovie_squad.roberta.by_thatdramebaazguy").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_movie_roberta_MITmovie_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/thatdramebaazguy/movie-roberta-MITmovie-squad --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from Davlan) author: John Snow Labs name: xlmroberta_ner_base_masakhan date: 2022-08-01 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-masakhaner` is a Multilingual model originally trained by `Davlan`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_masakhan_xx_4.1.0_3.0_1659356472350.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_masakhan_xx_4.1.0_3.0_1659356472350.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_masakhan","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_masakhan","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_masakhan| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|801.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Davlan/xlm-roberta-base-masakhaner - https://github.com/masakhane-io/masakhane-ner - https://arxiv.org/abs/2103.11811 --- layout: model title: Financial English Bert Embeddings (Base, Communication texts) author: John Snow Labs name: bert_embeddings_finbert_pretrain_yiyanghkust date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Financial English Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `finbert-pretrain-yiyanghkust` is a English model orginally available in Hugging Face as `yiyanghkust/finbert-pretrain`. It was trained on the following datasets: - Corporate Reports 10-K & 10-Q: 2.5B tokens - Earnings Call Transcripts: 1.3B tokens - Analyst Reports: 1.1B tokens {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_finbert_pretrain_yiyanghkust_en_3.4.2_3.0_1649672533001.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_finbert_pretrain_yiyanghkust_en_3.4.2_3.0_1649672533001.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_finbert_pretrain_yiyanghkust| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|412.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/philschmid/finbert-pretrain-yiyanghkust - https://arxiv.org/abs/2006.08097 --- layout: model title: Maltese CamemBert Embeddings (from fenrhjen) author: John Snow Labs name: camembert_embeddings_camembert_aux_amandes date: 2022-05-31 tags: [mt, open_source, camembert, embeddings] task: Embeddings language: mt edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `camembert_aux_amandes` is a Maltese model orginally trained by `fenrhjen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_camembert_aux_amandes_mt_3.4.4_3.0_1653985705496.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_camembert_aux_amandes_mt_3.4.4_3.0_1653985705496.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_camembert_aux_amandes| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|mt| |Size:|415.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/fenrhjen/camembert_aux_amandes --- layout: model title: Spanish RoBERTa Embeddings (Base, Stepwise) author: John Snow Labs name: roberta_embeddings_bertin_base_stepwise date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bertin-base-stepwise` is a Spanish model orginally trained by `bertin-project`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_stepwise_es_3.4.2_3.0_1649946195172.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_stepwise_es_3.4.2_3.0_1649946195172.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bertin_base_stepwise| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|234.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/bertin-project/bertin-base-stepwise --- layout: model title: Fast Neural Machine Translation Model from English to Indo-Iranian Languages author: John Snow Labs name: opus_mt_en_iir date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, iir, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `iir` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_iir_xx_2.7.0_2.4_1609167251409.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_iir_xx_2.7.0_2.4_1609167251409.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_iir| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering Base Uncased model (from anas-awadalla) author: John Snow Labs name: bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-16-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4_en_4.0.0_3.0_1657184537116.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4_en_4.0.0_3.0_1657184537116.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_few_shot_k_16_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-16-finetuned-squad-seed-4 --- layout: model title: Smaller BERT Embeddings (L-6_H-768_A-12) author: John Snow Labs name: small_bert_L6_768 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L6_768_en_2.6.0_2.4_1598345125237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L6_768_en_2.6.0_2.4_1598345125237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L6_768", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L6_768", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L6_768').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L6_768_embeddings I [-0.2927809953689575, -0.1677704006433487, -0.... love [-0.23247751593589783, 0.2524709105491638, -0.... NLP [0.03696206957101822, -0.11152120679616928, -0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L6_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_hady TFWav2Vec2ForCTC from hady author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_hady` is a English model originally trained by hady. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady_en_4.2.0_3.0_1664039200389.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady_en_4.2.0_3.0_1664039200389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_hady| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Recognize Entities DL Pipeline for Swedish - Small author: John Snow Labs name: entity_recognizer_sm date: 2021-03-22 tags: [open_source, swedish, entity_recognizer_sm, pipeline, sv] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: sv edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_sv_3.0.0_3.0_1616443122813.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_sv_3.0.0_3.0_1616443122813.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_sm', lang = 'sv') annotations = pipeline.fullAnnotate(""Hej från John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_sm", lang = "sv") val result = pipeline.fullAnnotate("Hej från John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hej från John Snow Labs! ""] result_df = nlu.load('sv.ner').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:------------------------------|:-----------------------------|:-----------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hej från John Snow Labs! '] | ['Hej från John Snow Labs!'] | ['Hej', 'från', 'John', 'Snow', 'Labs!'] | [[0.0306969992816448,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|sv| --- layout: model title: German T5ForConditionalGeneration Small Cased model (from Shahm) author: John Snow Labs name: t5_small_german date: 2023-01-31 tags: [de, open_source, t5, tensorflow] task: Text Generation language: de edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-german` is a German model originally trained by `Shahm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_german_de_4.3.0_3.0_1675126257283.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_german_de_4.3.0_3.0_1675126257283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_german","de") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_german","de") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_german| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|de| |Size:|285.5 MB| ## References - https://huggingface.co/Shahm/t5-small-german - https://paperswithcode.com/sota?task=Summarization&dataset=mlsum+de --- layout: model title: English asr_wav2vec2_large_in_lm TFWav2Vec2ForCTC from crossdelenna author: John Snow Labs name: pipeline_asr_wav2vec2_large_in_lm date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_in_lm` is a English model originally trained by crossdelenna. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_in_lm_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_in_lm_en_4.2.0_3.0_1664103269999.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_in_lm_en_4.2.0_3.0_1664103269999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_in_lm', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_in_lm", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_in_lm| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBERT Embeddings (%90 sparse) author: John Snow Labs name: distilbert_embeddings_distilbert_base_uncased_sparse_90_unstructured_pruneofa date: 2022-04-12 tags: [distilbert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-uncased-sparse-90-unstructured-pruneofa` is a English model orginally trained by `Intel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_uncased_sparse_90_unstructured_pruneofa_en_3.4.2_3.0_1649783362329.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_uncased_sparse_90_unstructured_pruneofa_en_3.4.2_3.0_1649783362329.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_uncased_sparse_90_unstructured_pruneofa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|123.7 MB| |Case sensitive:|false| ## References - https://huggingface.co/Intel/distilbert-base-uncased-sparse-90-unstructured-pruneofa - https://arxiv.org/abs/2111.05754 - https://github.com/IntelLabs/Model-Compression-Research-Package/tree/main/research/prune-once-for-all --- layout: model title: Detect PHI for Deidentification purposes (French, reduced entities) author: John Snow Labs name: ner_deid_generic date: 2022-02-11 tags: [deid, fr, licensed] task: De-identification language: fr edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM, CNN. Deidentification NER (French) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 7 entities. This NER model is trained with a custom dataset internally annotated, the [French WikiNER dataset](https://metatext.io/datasets/wikiner), a [public dataset of French company names](https://www.data.gouv.fr/fr/datasets/entreprises-immatriculees-en-2017/), a [public dataset of French hospital names](https://salesdorado.com/fichiers-prospection/hopitaux/) and several data augmentation mechanisms. ## Predicted Entities `CONTACT`, `NAME`, `DATE`, `ID`, `LOCATION`, `PROFESSION`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_fr_3.4.1_2.4_1644591444704.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_fr_3.4.1_2.4_1644591444704.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------+----------+ | token| ner_label| +------------+----------+ | J'ai| O| | vu| O| | en| O| |consultation| O| | Michel| B-NAME| | Martinez| I-NAME| | (| O| | 49| B-AGE| | ans| I-AGE| | )| O| | adressé| O| | au| O| | Centre|B-LOCATION| | Hospitalier|I-LOCATION| | De|I-LOCATION| | Plaisir|I-LOCATION| | pour| O| | un| O| | diabète| O| | mal| O| | contrôlé| O| | avec| O| | des| O| | symptômes| O| | datant| O| | de| O| | Mars| B-DATE| | 2015| I-DATE| | .| O| +------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|fr| |Size:|15.0 MB| ## References - Internal JSL annotated corpus - [French WikiNER dataset](https://metatext.io/datasets/wikiner) - [Public dataset of French company names](https://www.data.gouv.fr/fr/datasets/entreprises-immatriculees-en-2017/) - [Public dataset of French hospital names](https://salesdorado.com/fichiers-prospection/hopitaux/) ## Benchmarking ```bash label tp fp fn total precision recall f1 CONTACT 159.0 0.0 1.0 160.0 1.0 0.9938 0.9969 NAME 2633.0 111.0 197.0 2830.0 0.9595 0.9304 0.9447 DATE 2612.0 32.0 42.0 2654.0 0.9879 0.9842 0.986 ID 95.0 8.0 7.0 102.0 0.9223 0.9314 0.9268 LOCATION 3450.0 480.0 522.0 3972.0 0.8779 0.8686 0.8732 PROFESSION 326.0 54.0 82.0 408.0 0.8579 0.799 0.8274 AGE 395.0 37.0 46.0 441.0 0.9144 0.8957 0.9049 macro - - - - - - 0.9229 micro - - - - - - 0.9226 ``` --- layout: model title: German T5ForConditionalGeneration Small Cased model (from ml6team) author: John Snow Labs name: t5_mt5_small_german_finetune_mlsum date: 2023-01-30 tags: [de, open_source, t5, tensorflow] task: Text Generation language: de edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mt5-small-german-finetune-mlsum` is a German model originally trained by `ml6team`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_mt5_small_german_finetune_mlsum_de_4.3.0_3.0_1675106313224.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_mt5_small_german_finetune_mlsum_de_4.3.0_3.0_1675106313224.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_mt5_small_german_finetune_mlsum","de") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_mt5_small_german_finetune_mlsum","de") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_mt5_small_german_finetune_mlsum| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|de| |Size:|1.3 GB| ## References - https://huggingface.co/ml6team/mt5-small-german-finetune-mlsum - https://github.com/pltrdy/rouge --- layout: model title: Finance NER (10Q, lg, 139 entities) author: John Snow Labs name: finner_10q_xlbr date: 2022-12-02 tags: [10q, xlbr, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is an NER model containing 139 numeric financial entities from different 10Q reports. The tokens being annotated are the amounts, not any other surrounding word, but the context will determine what kind of amount is from the list of the 139 available. This is a large (`lg`) model, trained with 200K sentences. ## Predicted Entities `DeferredFinanceCostsNet`, `DisposalGroupIncludingDiscontinuedOperationConsideration`, `DebtInstrumentCarryingAmount`, `CommonStockSharesAuthorized`, `RestructuringCharges`, `DeferredFinanceCostsGross`, `OperatingLeasesRentExpenseNet`, `EquityMethodInvestmentOwnershipPercentage`, `ClassOfWarrantOrRightExercisePriceOfWarrantsOrRights1`, `DebtInstrumentTerm`, `DebtInstrumentRedemptionPricePercentage`, `CommonStockCapitalSharesReservedForFutureIssuance`, `LossContingencyAccrualAtCarryingValue`, `SaleOfStockPricePerShare`, `MinorityInterestOwnershipPercentageByParent`, `PropertyPlantAndEquipmentUsefulLife`, `TreasuryStockAcquiredAverageCostPerShare`, `Goodwill`, `SupplementalInformationForPropertyCasualtyInsuranceUnderwritersPriorYearClaimsAndClaimsAdjustmentExpense`, `CommonStockParOrStatedValuePerShare`, `OperatingLeaseWeightedAverageDiscountRatePercent`, `DebtInstrumentConvertibleConversionPrice1`, `AmortizationOfIntangibleAssets`, `PreferredStockSharesAuthorized`, `OperatingLeasePayments`, `DebtInstrumentMaturityDate`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodWeightedAverageGrantDateFairValue`, `EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate`, `AllocatedShareBasedCompensationExpense`, `PreferredStockDividendRatePercentage`, `StockRepurchaseProgramRemainingAuthorizedRepurchaseAmount1`, `TreasuryStockValueAcquiredCostMethod`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsVestedInPeriodTotalFairValue`, `IncomeTaxExpenseBenefit`, `DerivativeFixedInterestRate`, `RelatedPartyTransactionExpensesFromTransactionsWithRelatedParty`, `PublicUtilitiesRequestedRateIncreaseDecreaseAmount`, `RestructuringAndRelatedCostExpectedCost1`, `StockRepurchaseProgramAuthorizedAmount1`, `ShareBasedCompensation`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriodWeightedAverageGrantDateFairValue`, `LongTermDebtFairValue`, `LineOfCreditFacilityUnusedCapacityCommitmentFeePercentage`, `LineOfCreditFacilityCurrentBorrowingCapacity`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1`, `SharebasedCompensationArrangementBySharebasedPaymentAwardAwardVestingRightsPercentage`, `PaymentsToAcquireBusinessesGross`, `MinorityInterestOwnershipPercentageByNoncontrollingOwners`, `AntidilutiveSecuritiesExcludedFromComputationOfEarningsPerShareAmount`, `NumberOfReportableSegments`, `BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedIntangibleAssetsOtherThanGoodwill`, `OperatingLeaseCost`, `BusinessCombinationConsiderationTransferred1`, `UnrecognizedTaxBenefitsThatWouldImpactEffectiveTaxRate`, `CommonStockDividendsPerShareDeclared`, `AreaOfRealEstateProperty`, `LesseeOperatingLeaseTermOfContract`, `RevenueRemainingPerformanceObligation`, `RelatedPartyTransactionAmountsOfTransaction`, `InterestExpense`, `OperatingLeaseExpense`, `StockIssuedDuringPeriodSharesNewIssues`, `DebtInstrumentFaceAmount`, `CapitalizedContractCostAmortization`, `DebtInstrumentBasisSpreadOnVariableRate1`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsNonvestedNumber`, `GainsLossesOnExtinguishmentOfDebt`, `LineOfCreditFacilityRemainingBorrowingCapacity`, `OperatingLeaseRightOfUseAsset`, `OperatingLeaseWeightedAverageRemainingLeaseTerm1`, `OperatingLossCarryforwards`, `ConcentrationRiskPercentage1`, `GuaranteeObligationsMaximumExposure`, `StockRepurchasedAndRetiredDuringPeriodShares`, `LesseeOperatingLeaseRenewalTerm`, `ContractWithCustomerLiabilityRevenueRecognized`, `DefinedBenefitPlanContributionsByEmployer`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross`, `RepaymentsOfDebt`, `EmployeeServiceShareBasedCompensationNonvestedAwardsTotalCompensationCostNotYetRecognized`, `BusinessAcquisitionPercentageOfVotingInterestsAcquired`, `DebtInstrumentInterestRateEffectivePercentage`, `AcquiredFiniteLivedIntangibleAssetsWeightedAverageUsefulLife`, `DebtInstrumentUnamortizedDiscount`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAuthorized`, `BusinessCombinationContingentConsiderationLiability`, `DebtInstrumentInterestRateStatedPercentage`, `LeaseAndRentalExpense`, `RevenueFromContractWithCustomerExcludingAssessedTax`, `SharePrice`, `CommonStockSharesOutstanding`, `ContractWithCustomerLiability`, `DerivativeNotionalAmount`, `RevenueFromRelatedParties`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsExercisesInPeriodTotalIntrinsicValue`, `Revenues`, `EmployeeServiceShareBasedCompensationNonvestedAwardsTotalCompensationCostNotYetRecognizedShareBasedAwardsOtherThanOptions`, `AccrualForEnvironmentalLossContingencies`, `ProceedsFromIssuanceOfCommonStock`, `EmployeeServiceShareBasedCompensationTaxBenefitFromCompensationExpense`, `IncomeLossFromEquityMethodInvestments`, `NumberOfOperatingSegments`, `UnrecognizedTaxBenefits`, `RevenueFromContractWithCustomerIncludingAssessedTax`, `LossContingencyDamagesSoughtValue`, `SharebasedCompensationArrangementBySharebasedPaymentAwardExpirationPeriod`, `TreasuryStockSharesAcquired`, `FiniteLivedIntangibleAssetUsefulLife`, `BusinessCombinationRecognizedIdentifiableAssetsAcquiredAndLiabilitiesAssumedIntangibles`, `EffectiveIncomeTaxRateContinuingOperations`, `LossContingencyEstimateOfPossibleLoss`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardNumberOfSharesAvailableForGrant`, `BusinessCombinationAcquisitionRelatedCosts`, `StockRepurchasedDuringPeriodShares`, `CashAndCashEquivalentsFairValueDisclosure`, `LineOfCreditFacilityInterestRateAtPeriodEnd`, `ShareBasedCompensationArrangementByShareBasedPaymentAwardEquityInstrumentsOtherThanOptionsGrantsInPeriod`, `CumulativeEffectOfNewAccountingPrincipleInPeriodOfAdoption`, `LettersOfCreditOutstandingAmount`, `EmployeeServiceShareBasedCompensationNonvestedAwardsTotalCompensationCostNotYetRecognizedPeriodForRecognition1`, `NumberOfRealEstateProperties`, `DebtWeightedAverageInterestRate`, `SaleOfStockNumberOfSharesIssuedInTransaction`, `AssetImpairmentCharges`, `Depreciation`, `DebtInstrumentFairValue`, `DefinedContributionPlanCostRecognized`, `InterestExpenseDebt`, `LossContingencyPendingClaimsNumber`, `PaymentsToAcquireBusinessesNetOfCashAcquired`, `BusinessAcquisitionEquityInterestsIssuedOrIssuableNumberOfSharesIssued`, `GoodwillImpairmentLoss`, `LineOfCredit`, `AmortizationOfFinancingCosts`, `EquityMethodInvestments`, `LineOfCreditFacilityCommitmentFeePercentage`, `LongTermDebt`, `LineOfCreditFacilityMaximumBorrowingCapacity`, `OperatingLeaseLiability` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_10Q_XLBR){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_10q_xlbr_en_1.0.0_3.0_1669977147020.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_10q_xlbr_en_1.0.0_3.0_1669977147020.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) nerTagger = finance.NerModel.pretrained('finner_10q_xlbr', 'en', 'finance/models')\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") pipeline = nlp.Pipeline(stages=[documentAssembler, sentence, tokenizer, embeddings, ner_model ]) text = """Common Stock The authorized capital of the Company is 200,000,000 common shares , par value $ 0.001 , of which 12,481,724 are issued or outstanding .""" df = spark.createDataFrame([[text]]).toDF("text") fit = pipeline.fit(df) result = fit.transform(df) result_df = result.select(F.explode(F.arrays_zip(result.token.result,result.ner.result, result.ner.metadata)).alias("cols"))\ .select(F.expr("cols['0']").alias("token"), F.expr("cols['1']").alias("ner_label"), F.expr("cols['2']['confidence']").alias("confidence")) result_df.show(50, truncate=100) ```

## Results ```bash +-----------+-------------------------------------+----------+ | token| ner_label|confidence| +-----------+-------------------------------------+----------+ | Common| O| 1.0| | Stock| O| 1.0| | The| O| 1.0| | authorized| O| 1.0| | capital| O| 1.0| | of| O| 1.0| | the| O| 1.0| | Company| O| 1.0| | is| O| 1.0| |200,000,000| B-CommonStockSharesAuthorized| 0.9932| | common| O| 1.0| | shares| O| 1.0| | ,| O| 1.0| | par| O| 1.0| | value| O| 1.0| | $| O| 1.0| | 0.001|B-CommonStockParOrStatedValuePerShare| 0.9988| | ,| O| 1.0| | of| O| 1.0| | which| O| 1.0| | 12,481,724| B-CommonStockSharesOutstanding| 0.9649| | are| O| 1.0| | issued| O| 1.0| | or| O| 1.0| |outstanding| O| 1.0| | .| O| 1.0| +-----------+-------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_10q_xlbr| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|17.0 MB| ## References An in-house modified version of https://huggingface.co/datasets/nlpaueb/finer-139, re-splited and filtered to focus on sentences with bigger density of tags. ## Benchmarking ```bash label tp fp fn prec rec f1 Macro-average 53613 10309 10243 0.8324958 0.8049274 0.8184795 Micro-average 53613 10309 10243 0.8387253 0.8395922 0.8391586 ``` --- layout: model title: Ganda asr_wav2vec2_large_xlsr_luganda TFWav2Vec2ForCTC from lucio author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_luganda date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_luganda` is a Ganda model originally trained by lucio. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_luganda_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_luganda_lg_4.2.0_3.0_1664036777655.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_luganda_lg_4.2.0_3.0_1664036777655.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_luganda', lang = 'lg') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_luganda", lang = "lg") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_luganda| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|lg| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Tai to English Pipeline author: John Snow Labs name: translate_taw_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, taw, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `taw` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_taw_en_xx_2.7.0_2.4_1609700707816.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_taw_en_xx_2.7.0_2.4_1609700707816.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_taw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_taw_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.taw.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_taw_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from husnu) author: John Snow Labs name: bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xtremedistil-l6-h256-uncased-finetuned_lr-2e-05_epochs-3` is a English model orginally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3_en_4.0.0_3.0_1654192622134.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3_en_4.0.0_3.0_1654192622134.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.xtremedistiled_uncased_lr_2e_05_epochs_3.by_husnu").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_xtremedistil_l6_h256_uncased_finetuned_lr_2e_05_epochs_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|47.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/husnu/xtremedistil-l6-h256-uncased-finetuned_lr-2e-05_epochs-3 --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Dutch (WikiNER 840B 300) author: John Snow Labs name: wikiner_840B_300 date: 2020-05-10 task: Named Entity Recognition language: nl edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [ner, nl, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 840B 300 is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_NL){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_NL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_nl_2.5.0_2.4_1588546201484.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_nl_2.5.0_2.4_1588546201484.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates III |PER | |Amerikaanse |MISC | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |CEO |ORG | |Gates |PER | |Seattle |LOC | |Washington |LOC | |Paul Allen Microsoft |PER | |Albuquerque |LOC | |New Mexico |LOC | |Gates |PER | |CEO |ORG | |CEO |ORG | |Eind |ORG | |Gates |PER | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|ORG | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|nl| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://nl.wikipedia.org](https://nl.wikipedia.org) --- layout: model title: Legal Executive Employment Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_executive_employment_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, executive_employment, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_executive_employment_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `executive-employment-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `executive-employment-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_executive_employment_agreement_bert_en_1.0.0_3.0_1669313118478.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_executive_employment_agreement_bert_en_1.0.0_3.0_1669313118478.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_executive_employment_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[executive-employment-agreement]| |[other]| |[other]| |[executive-employment-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_executive_employment_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support executive-employment-agreement 0.92 0.94 0.93 36 other 0.97 0.95 0.96 65 accuracy - - 0.95 101 macro-avg 0.94 0.95 0.95 101 weighted-avg 0.95 0.95 0.95 101 ``` --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten` is a English model originally trained by patrickvonplaten. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_en_4.2.0_3.0_1664105259391.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten_en_4.2.0_3.0_1664105259391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_patrickvonplaten| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect Diagnoses and Procedures (Spanish) author: John Snow Labs name: ner_diag_proc class: NerDLModel language: es repository: clinical/models date: 2020-07-08 task: Named Entity Recognition edition: Healthcare NLP 2.5.3 spark_version: 2.4 tags: [clinical,licensed,ner,es] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Pretrained named entity recognition deep learning model for diagnostics and procedures in spanish ## Predicted Entities ``DIAGNOSTICO``, ``PROCEDIMIENTO``. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diag_proc_es_2.5.3_2.4_1594168623415.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diag_proc_es_2.5.3_2.4_1594168623415.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embed = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") model = NerDLModel.pretrained("ner_diag_proc","es","clinical/models")\ .setInputCols("sentence","token","word_embeddings")\ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embed, model, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.']], ["text"])) ``` ```scala ... val embed = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") val model = NerDLModel.pretrained("ner_diag_proc","es","clinical/models") .setInputCols("sentence","token","word_embeddings") .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embed, model, ner_converter)) val data = Seq("HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner").predict("""HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.""") ```

## Results ```bash +----------------------+-------------+ |chunk |ner_label | +----------------------+-------------+ |ENFERMEDAD |DIAGNOSTICO | |cáncer de vejiga |DIAGNOSTICO | |resección |PROCEDIMIENTO| |cistectomía |PROCEDIMIENTO| |estrés |DIAGNOSTICO | |infarto subendocárdico|DIAGNOSTICO | |enfermedad |DIAGNOSTICO | |arterias coronarias |DIAGNOSTICO | +----------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|----------------------------------| | Name: | ner_diag_proc | | Type: | NerDLModel | | Compatibility: | 2.5.3 | | License: | Licensed | | Edition: | Official | |Input labels: | [sentence, token, word_embeddings] | |Output labels: | [ner] | | Language: | es | | Dependencies: | embeddings_scielowiki_300d | {:.h2_title} ## Data Source Trained on CodiEsp Challenge dataset trained with `embeddings_scielowiki_300d` https://temu.bsc.es/codiesp/ ## Benchmarking ```bash +-------------+------+------+------+------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +-------------+------+------+------+------+---------+------+------+ |PROCEDIMIENTO|2299.0|1103.0| 860.0|3159.0| 0.6758|0.7278|0.7008| | DIAGNOSTICO|6623.0|1364.0|2974.0|9597.0| 0.8292|0.6901|0.7533| +-------------+------+------+------+------+---------+------+------+ +------------------+ | macro| +------------------+ |0.7270531284138397| +------------------+ +------------------+ | micro| +------------------+ |0.7402992400932049| +------------------+ ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from silveto) author: John Snow Labs name: distilbert_qa_silveto_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `silveto`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_silveto_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772771581.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_silveto_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772771581.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_silveto_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_silveto_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_silveto_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/silveto/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate English to Uralic languages Pipeline author: John Snow Labs name: translate_en_urj date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, urj, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `urj` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_urj_xx_2.7.0_2.4_1609687665593.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_urj_xx_2.7.0_2.4_1609687665593.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_urj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_urj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.urj').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_urj| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Albanian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, sq, open_source] task: Embeddings language: sq edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sq_3.4.1_3.0_1647282075905.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sq_3.4.1_3.0_1647282075905.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash {'mesh': ['C028491', 'D019326', 'C579867'], 'umls': ['C0970275', 'C0886627', 'C3696376']} Note: | MeSH | Details | | ---------- | ----------------------------:| | C028491 | 1,3-butylene glycol | | D019326 | 17-alpha-Hydroxyprogesterone | | C579867 | 3-Methylglutaconic Aciduria | | UMLS | Details | | ---------- | ---------------------------:| | C0970275 | 1,3-butylene glycol | | C0886627 | 17-hydroxyprogesterone | | C3696376 | 3-methylglutaconic aciduria | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|mesh_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: English asr_model_sid_voxforge_cetuc_1 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: asr_model_sid_voxforge_cetuc_1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_sid_voxforge_cetuc_1` is a English model originally trained by joaoalvarenga. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_model_sid_voxforge_cetuc_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_model_sid_voxforge_cetuc_1_en_4.2.0_3.0_1664021832928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_model_sid_voxforge_cetuc_1_en_4.2.0_3.0_1664021832928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_model_sid_voxforge_cetuc_1", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_model_sid_voxforge_cetuc_1", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_model_sid_voxforge_cetuc_1| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect PHI in text (ner_deid_sd_larg) author: John Snow Labs name: ner_deid_sd_large date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect PHI in text for de-identification using pretrained NER model. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `PROFESSION`, `CONTACT`, `DATE`, `NAME`, `AGE`, `ID`, `LOCATION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_en_3.0.0_3.0_1617260861713.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_sd_large_en_3.0.0_3.0_1617260861713.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_deid_sd_large", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentenceDetector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) sample_text = """Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""" df = spark.createDataFrame([[sample_text]]).toDF("text") result = nlpPipeline.fit(df).transform(df) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_deid_sd_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter)) val data = Seq("""Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.sd_large").predict("""Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years old , Record date : 2079-11-09 . Cocke County Baptist Hospital , 0295 Keats Street , Phone 302-786-5227.""") ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |2093-01-13 |DATE | |David Hale |NAME | |Hendrickson Ora |NAME | |7194334 |ID | |01/13/93 |DATE | |Oliveira |NAME | |25 |AGE | |2079-11-09 |DATE | |Cocke County Baptist Hospital|LOCATION | |0295 Keats Street |LOCATION | |302-786-5227 |CONTACT | +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_sd_large| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab0_by_sherry7144 TFWav2Vec2ForCTC from sherry7144 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab0_by_sherry7144 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab0_by_sherry7144` is a English model originally trained by sherry7144. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab0_by_sherry7144_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab0_by_sherry7144_en_4.2.0_3.0_1664037814643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab0_by_sherry7144_en_4.2.0_3.0_1664037814643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab0_by_sherry7144", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab0_by_sherry7144", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab0_by_sherry7144| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from Gam) author: John Snow Labs name: roberta_qa_base_finetuned_cuad_gam date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-cuad-gam` is a English model originally trained by `Gam`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_cuad_gam_en_4.3.0_3.0_1674216483026.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_cuad_gam_en_4.3.0_3.0_1674216483026.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_cuad_gam","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_cuad_gam","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_cuad_gam| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|450.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Gam/roberta-base-finetuned-cuad-gam --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman` is a Finnish model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_fi_4.2.0_3.0_1664041651614.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman_fi_4.2.0_3.0_1664041651614.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_finnish_by_jonatasgrosman| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: Spanish Bert Embeddings (from mmaguero) author: John Snow Labs name: bert_embeddings_beto_gn_base_cased date: 2022-04-11 tags: [bert, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `beto-gn-base-cased` is a Spanish model orginally trained by `mmaguero`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_beto_gn_base_cased_es_3.4.2_3.0_1649671395707.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_beto_gn_base_cased_es_3.4.2_3.0_1649671395707.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_beto_gn_base_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|411.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/mmaguero/beto-gn-base-cased --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Modified_PubMedBERT date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Modified_PubMedBERT` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Modified_PubMedBERT_en_4.0.0_3.0_1657108813220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Modified_PubMedBERT_en_4.0.0_3.0_1657108813220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Modified_PubMedBERT| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Modified_PubMedBERT --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squad1.1-block-sparse-0.32-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1_en_4.0.0_3.0_1654181454017.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1_en_4.0.0_3.0_1654181454017.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.1.1_block_sparse_0.32_v1.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squad1.1_block_sparse_0.32_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|213.4 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squad1.1-block-sparse-0.32-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf - https://www.aclweb.org/anthology/N19-1423/ - https://arxiv.org/abs/2005.07683 --- layout: model title: Word2Vec Embeddings in Nepali (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, ne, open_source] task: Embeddings language: ne edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ne_3.4.1_3.0_1647448067646.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ne_3.4.1_3.0_1647448067646.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_hfl_chinese_wwm_ext","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_hfl_chinese_wwm_ext","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_hfl_chinese_wwm_ext| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-bert-wwm-ext - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English asr_wav2vec2_large_xlsr_sermon TFWav2Vec2ForCTC from sharonibejih author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_sermon date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_sermon` is a English model originally trained by sharonibejih. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_sermon_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_sermon_en_4.2.0_3.0_1664115294414.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_sermon_en_4.2.0_3.0_1664115294414.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_sermon', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_sermon", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_sermon| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Arabic BertForMaskedLM Large Cased model (from asafaya) author: John Snow Labs name: bert_embeddings_large_arabic date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-arabic` is a Arabic model originally trained by `asafaya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabic_ar_4.2.4_3.0_1670019914064.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabic_ar_4.2.4_3.0_1670019914064.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabic","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabic","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_arabic| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/asafaya/bert-large-arabic - https://traces1.inria.fr/oscar/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/bert - https://www.tensorflow.org/tfrc - https://github.com/alisafaya/Arabic-BERT --- layout: model title: Word2Vec Embeddings in Goan Konkani (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, gom, open_source] task: Embeddings language: gom edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_gom_3.4.1_3.0_1647374766723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_gom_3.4.1_3.0_1647374766723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_1zha5ono","en") \ .setInputCols(["document"]) \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_1zha5ono","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_1zha5ono| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|947.0 MB| ## References - https://huggingface.co/tscholak/1zha5ono - https://arxiv.org/abs/2109.05093 - https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#lm-adapted-t511lm100k - https://yale-lily.github.io/spider - https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#lm-adapted-t511lm100k - https://github.com/ElementAI/picard - https://github.com/ElementAI/picard - https://arxiv.org/abs/2109.05093 - https://github.com/ElementAI/picard --- layout: model title: ELECTRA Embeddings(ELECTRA Small) author: John Snow Labs name: electra_small_uncased date: 2020-08-27 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ELECTRA is a BERT-like model that is pre-trained as a discriminator in a set-up resembling a generative adversarial network (GAN). It was originally published by: Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning: [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/forum?id=r1xMH1BtvB), ICLR 2020. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_small_uncased_en_2.6.0_2.4_1598485458536.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_small_uncased_en_2.6.0_2.4_1598485458536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("electra_small_uncased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("electra_small_uncased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.electra.small_uncased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_electra_small_uncased_embeddings token [0.7677724361419678, -0.621893048286438, -0.93... I [1.1656712293624878, -0.04712940752506256, 0.1... love [1.0845087766647339, -0.00533430278301239, -0.... NLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_small_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/electra_small/2](https://tfhub.dev/google/electra_small/2) --- layout: model title: Explain Document Pipeline for Danish author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, danish, explain_document_md, pipeline, da] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: da edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_da_3.0.0_3.0_1616437187649.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_da_3.0.0_3.0_1616437187649.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'da') annotations = pipeline.fullAnnotate(""Hej fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "da") val result = pipeline.fullAnnotate("Hej fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hej fra John Snow Labs! ""] result_df = nlu.load('da.explain.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hej fra John Snow Labs! '] | ['Hej fra John Snow Labs!'] | ['Hej', 'fra', 'John', 'Snow', 'Labs!'] | ['Hej', 'fra', 'John', 'Snow', 'Labs!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.4006600081920624,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|da| --- layout: model title: Dutch Part of Speech Tagger (from oliverguhr) author: John Snow Labs name: roberta_pos_fullstop_dutch_punctuation_prediction date: 2022-05-03 tags: [roberta, pos, part_of_speech, nl, open_source] task: Part of Speech Tagging language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `fullstop-dutch-punctuation-prediction` is a Dutch model orginally trained by `oliverguhr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_pos_fullstop_dutch_punctuation_prediction_nl_3.4.2_3.0_1651596311447.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_pos_fullstop_dutch_punctuation_prediction_nl_3.4.2_3.0_1651596311447.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_fullstop_dutch_punctuation_prediction| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|nl| |Size:|436.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/oliverguhr/fullstop-dutch-punctuation-prediction --- layout: model title: German BertForTokenClassification Cased model (from elenanereiss) author: John Snow Labs name: bert_token_classifier_german_ler date: 2022-11-30 tags: [de, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-german-ler` is a German model originally trained by `elenanereiss`. ## Predicted Entities ``, `EUN`, `LIT`, `RR`, `INN`, `RS`, `PER`, `VO`, `UN`, `MRK`, `AN`, `LD`, `STR`, `GRT`, `ORG`, `GS`, `VT`, `LDS`, `ST`, `VS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_german_ler_de_4.2.4_3.0_1669815202449.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_german_ler_de_4.2.4_3.0_1669815202449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_german_ler","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_german_ler","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_german_ler| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|de| |Size:|407.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/elenanereiss/bert-german-ler - https://github.com/elenanereiss/bert-legal-ner - https://paperswithcode.com/sota?task=Token+Classification&dataset=elenanereiss%2Fgerman-ler --- layout: model title: Latin BertForMaskedLM Cased model (from cook) author: John Snow Labs name: bert_embeddings_cicero_similis date: 2022-12-02 tags: [la, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: la edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cicero-similis` is a Latin model originally trained by `cook`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_cicero_similis_la_4.2.4_3.0_1670021953584.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_cicero_similis_la_4.2.4_3.0_1670021953584.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_cicero_similis","la") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_cicero_similis","la") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_cicero_similis| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|la| |Size:|233.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/cook/cicero-similis --- layout: model title: Fast Neural Machine Translation Model from English to Kirundi author: John Snow Labs name: opus_mt_en_rn date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, rn, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `rn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_rn_xx_2.7.0_2.4_1609170714678.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_rn_xx_2.7.0_2.4_1609170714678.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_rn| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForSequenceClassification Base Cased model (from mrm8488) author: John Snow Labs name: roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod date: 2022-07-13 tags: [en, open_source, roberta, sequence_classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `codebert-base-finetuned-detect-insecure-code` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod_en_4.0.0_3.0_1657716286514.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod_en_4.0.0_3.0_1657716286514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_sequence_classifier_codebert_base_finetuned_detect_insecure_cod| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|469.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/codebert-base-finetuned-detect-insecure-code - https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/Defect-detection - https://arxiv.org/pdf/1907.11692.pdf - https://arxiv.org/pdf/2002.08155.pdf --- layout: model title: Extract Health and Behaviours Problems Entities from Social Determinants of Health Texts author: John Snow Labs name: ner_sdoh_health_behaviours_problems_wip date: 2023-02-24 tags: [clinical, licensed, en, social_determinants, ner, public_health, sdoh, health, behaviours, problems, health_behaviours_problems] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts health and behaviours problems related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Diet`, `Mental_Health`, `Obesity`, `Eating_Disorder`, `Sexual_Activity`, `Disability`, `Quality_Of_Life`, `Other_Disease`, `Exercise`, `Communicable_Disease`, `Hyperlipidemia`, `Hypertension` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_health_behaviours_problems_wip_en_4.3.1_3.0_1677198610586.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_health_behaviours_problems_wip_en_4.3.1_3.0_1677198610586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_health_behaviours_problems_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = ["She has not been getting regular exercise and not followed diet for approximately two years due to chronic sciatic pain.", "Medical History: The patient is a 32-year-old female who presents with a history of anxiety, depression, bulimia nervosa, elevated cholesterol, and substance abuse.", "Pt was intubated atthe scene & currently sedated due to high BP. Also, he is currently on social security disability."] data = spark.createDataFrame(sample_texts, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_health_behaviours_problems_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("She has not been getting regular exercise for approximately two years due to chronic sciatic pain.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +--------------------+-----+---+---------------+ |chunk |begin|end|ner_label | +--------------------+-----+---+---------------+ |regular exercise |25 |40 |Exercise | |diet |59 |62 |Diet | |chronic sciatic pain|99 |118|Other_Disease | |anxiety |84 |90 |Mental_Health | |depression |93 |102|Mental_Health | |bulimia nervosa |105 |119|Eating_Disorder| |elevated cholesterol|122 |141|Hyperlipidemia | |high BP |56 |62 |Hypertension | |disability |106 |115|Disability | +--------------------+-----+---+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_health_behaviours_problems_wip| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.0 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Quality_Of_Life 127.0 19.0 3.0 130.0 0.869863 0.976923 0.920290 Eating_Disorder 56.0 5.0 0.0 56.0 0.918033 1.000000 0.957265 Obesity 16.0 2.0 7.0 23.0 0.888889 0.695652 0.780488 Exercise 103.0 6.0 5.0 108.0 0.944954 0.953704 0.949309 Communicable_Disease 61.0 11.0 5.0 66.0 0.847222 0.924242 0.884058 Hypertension 52.0 0.0 2.0 54.0 1.000000 0.962963 0.981132 Other_Disease 1068.0 85.0 79.0 1147.0 0.926279 0.931125 0.928696 Diet 66.0 12.0 15.0 81.0 0.846154 0.814815 0.830189 Disability 95.0 1.0 6.0 101.0 0.989583 0.940594 0.964467 Mental_Health 1020.0 45.0 134.0 1154.0 0.957746 0.883882 0.919333 Hyperlipidemia 19.0 1.0 2.0 21.0 0.950000 0.904762 0.926829 Sexual_Activity 82.0 15.0 6.0 88.0 0.845361 0.931818 0.886486 ``` --- layout: model title: Relation Extraction between generic entities author: John Snow Labs name: generic_re date: 2022-12-20 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.3 spark_version: 3.0 tags: [re, en, clinical, licensed, relation extraction, generic] supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description We already have more than 80 relation extraction (RE) models that can extract relations between certain named entities. Nevertheless, there are some rare entities or cases that you may not find the right RE or the one you find may not work as expected due to nature of your dataset. In order to ease this burden, we are releasing a generic RE model (generic_re) that can be used between any named entities using the syntactic distances, POS tags and dependency tree between the entities. You can tune this model by using the setMaxSyntacticDistance param. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_POSOLOGY/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = sparknlp.annotators.Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") words_embedder = WordEmbeddingsModel()\ .pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("embeddings") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") ner_tagger = MedicalNerModel()\ .pretrained("ner_oncology_wip", "en", "clinical/models")\ .setInputCols("sentences", "tokens", "embeddings")\ .setOutputCol("ner_tags") ner_chunker = NerConverterInternal()\ .setInputCols(["sentences", "tokens", "ner_tags"])\ .setOutputCol("ner_chunks") dependency_parser = DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") reModel = RelationExtractionModel()\ .pretrained("generic_re")\ .setInputCols(["embeddings", "pos_tags", "ner_chunks", "dependencies"])\ .setOutputCol("relations")\ .setRelationPairs(["Biomarker-Biomarker_Result", "Biomarker_Result-Biomarker", "Oncogene-Biomarker_Result", "Biomarker_Result-Oncogene", "Pathology_Test-Pathology_Result", "Pathology_Result-Pathology_Test"]) \ .setMaxSyntacticDistance(4) pipeline = Pipeline(stages=[ documenter, sentencer, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_chunker, dependency_parser, reModel ]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = pipeline.fit(empty_data) text = """Pathology showed tumor cells, which were positive for estrogen and progesterone receptors.""" ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentences") tokenizer = Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val pos_tagger = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val ner_tagger = MedicalNerModel() .pretrained("ner_oncology_wip", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_chunker = new NerConverterInternal() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") val re_Model = RelationExtractionModel() .pretrained("generic_re") .setInputCols(Array("embeddings", "pos_tags", "ner_chunks", "dependencies")) .setOutputCol("relations") .setRelationPairs(Array("Biomarker-Biomarker_Result", "Biomarker_Result-Biomarker", "Oncogene-Biomarker_Result", "Biomarker_Result-Oncogene", "Pathology_Test-Pathology_Result", "Pathology_Result-Pathology_Test")) .setMaxSyntacticDistance(4) val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_chunker, dependecy_parser, re_Model)) val data = Seq("Pathology showed tumor cells, which were positive for estrogen and progesterone receptors.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash |sentence |entity1_begin |entity1_end | chunk1 | entity1 |entity2_begin |entity2_end | chunk2 | entity2 | relation |confidence| |--------:|-------------:|-----------:|:----------|:-----------------|-------------:|-----------:|:-----------------------|:-----------------|:--------------------------------|----------| | 0 | 1 | 9 | Pathology | Pathology_Test | 18 | 28 | tumor cells | Pathology_Result | Pathology_Test-Pathology_Result | 1| | 0 | 42 | 49 | positive | Biomarker_Result | 55 | 62 | estrogen | Biomarker | Biomarker_Result-Biomarker | 1| | 0 | 42 | 49 | positive | Biomarker_Result | 68 | 89 | progesterone receptors | Biomarker | Biomarker_Result-Biomarker | 1| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|generic_re| |Compatibility:|Healthcare NLP 4.2.3+| |Edition:|Official| |License:|Licensed| |Language:|[en]| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_ali221000262 TFWav2Vec2ForCTC from ali221000262 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_ali221000262 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_ali221000262` is a English model originally trained by ali221000262. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_ali221000262_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_ali221000262_en_4.2.0_3.0_1664036614909.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_ali221000262_en_4.2.0_3.0_1664036614909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_ali221000262", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_ali221000262", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_ali221000262| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: Vietnamese T5ForConditionalGeneration Base Cased model (from VietAI) author: John Snow Labs name: t5_vit5_base_vietnews_summarization date: 2023-01-31 tags: [vi, open_source, t5, tensorflow] task: Text Generation language: vi edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `vit5-base-vietnews-summarization` is a Vietnamese model originally trained by `VietAI`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_vit5_base_vietnews_summarization_vi_4.3.0_3.0_1675158454304.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_vit5_base_vietnews_summarization_vi_4.3.0_3.0_1675158454304.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_vit5_base_vietnews_summarization","vi") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_vit5_base_vietnews_summarization","vi") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_vit5_base_vietnews_summarization| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|vi| |Size:|485.0 MB| ## References - https://huggingface.co/VietAI/vit5-base-vietnews-summarization - https://paperswithcode.com/sota/abstractive-text-summarization-on-vietnews?p=vit5-pretrained-text-to-text-transformer-for - https://github.com/vietai/ViT5 - https://github.com/vietai/ViT5/blob/main/eval/Eval_vietnews_sum.ipynb --- layout: model title: English RobertaForQuestionAnswering (from huxxx657) author: John Snow Labs name: roberta_qa_roberta_base_finetuned_scrambled_squad_15_new date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-scrambled-squad-15-new` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_15_new_en_4.0.0_3.0_1655734119312.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_finetuned_scrambled_squad_15_new_en_4.0.0_3.0_1655734119312.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_15_new","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_finetuned_scrambled_squad_15_new","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_scrambled_15_v2.by_huxxx657").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_finetuned_scrambled_squad_15_new| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-scrambled-squad-15-new --- layout: model title: Legal Warrant Agreement Document Binary Classifier (Longformer) author: John Snow Labs name: legclf_warrant_agreement date: 2022-12-18 tags: [en, legal, classification, licensed, document, longformer, warrant, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_warrant_agreement` model is a Longformer Document Classifier used to classify if the document belongs to the class `warrant-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If your document needs to process more than 4096 tokens, you can try the following: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. ## Predicted Entities `warrant-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_warrant_agreement_en_1.0.0_3.0_1671393665185.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_warrant_agreement_en_1.0.0_3.0_1671393665185.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[warrant-agreement]| |[other]| |[other]| |[warrant-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_warrant_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.98 0.99 230 warrant-agreement 0.96 1.00 0.98 99 accuracy - - 0.99 329 macro-avg 0.98 0.99 0.99 329 weighted-avg 0.99 0.99 0.99 329 ``` --- layout: model title: Extract anatomical entities (Voice of the Patients) author: John Snow Labs name: ner_vop_anatomy_wip date: 2023-05-19 tags: [licensed, clinical, en, ner, vop, patient, anatomy] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts anatomical terms from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `BodyPart`, `Laterality` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_wip_en_4.4.2_3.0_1684511968875.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_anatomy_wip_en_4.4.2_3.0_1684511968875.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------|:------------| | muscle | BodyPart | | neck | BodyPart | | trapezius | BodyPart | | head | BodyPart | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_anatomy_wip| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 BodyPart 2727 221 173 2900 0.93 0.94 0.93 Laterality 557 81 71 628 0.87 0.89 0.88 macro_avg 3284 302 244 3528 0.90 0.92 0.90 micro_avg 3284 302 244 3528 0.92 0.93 0.92 ``` --- layout: model title: English BertForQuestionAnswering model (from recobo) author: John Snow Labs name: bert_qa_chemical_bert_uncased_squad2 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chemical-bert-uncased-squad2` is a English model orginally trained by `recobo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_chemical_bert_uncased_squad2_en_4.0.0_3.0_1654537776973.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_chemical_bert_uncased_squad2_en_4.0.0_3.0_1654537776973.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_chemical_bert_uncased_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_chemical_bert_uncased_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_chemical.bert.uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_chemical_bert_uncased_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/recobo/chemical-bert-uncased-squad2 --- layout: model title: Legal Binding Effects Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_binding_effects_bert date: 2023-03-05 tags: [en, legal, classification, clauses, binding_effects, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Binding_Effects` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Binding_Effects`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_binding_effects_bert_en_1.0.0_3.0_1678050516792.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_binding_effects_bert_en_1.0.0_3.0_1678050516792.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_binding_effects_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Binding_Effects]| |[Other]| |[Other]| |[Binding_Effects]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_binding_effects_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Binding_Effects 0.91 0.89 0.90 70 Other 0.92 0.94 0.93 97 accuracy - - 0.92 167 macro-avg 0.92 0.91 0.91 167 weighted-avg 0.92 0.92 0.92 167 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Ayoola) author: John Snow Labs name: distilbert_qa_ayoola_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Ayoola`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ayoola_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768287189.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ayoola_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768287189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ayoola_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ayoola_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ayoola_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Ayoola/distilbert-base-uncased-finetuned-squad --- layout: model title: Spanish Bert Embeddings (from Geotrend) author: John Snow Labs name: bert_embeddings_bert_base_es_cased date: 2022-04-11 tags: [bert, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-es-cased` is a Spanish model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_es_cased_es_3.4.2_3.0_1649671116551.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_es_cased_es_3.4.2_3.0_1649671116551.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_es_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|399.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-es-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Detect 10 Different Entities in Hebrew (hebrew_cc_300d embeddings) author: John Snow Labs name: hebrewner_cc_300d date: 2022-08-09 tags: [he, open_source] task: Named Entity Recognition language: he edition: Spark NLP 4.0.2 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses Hebrew word embeddings to find 10 different types of entities in Hebrew text. It is trained using `hebrew_cc_300d` word embeddings, please use the same embeddings in the pipeline. Predicted entities: Persons-`PERS`, Dates-`DATE`, Organizations-`ORG`, Locations-`LOC`, Percentage-`PERCENT`, Money-`MONEY`, Time-`TIME`, Miscellaneous (Affiliation)-`MISC_AFF`, Miscellaneous (Event)-`MISC_EVENT`, Miscellaneous (Entity)-`MISC_ENT`. ## Predicted Entities `PERS`, `LOC`, `DATE`, `ORG`, `TIME`, `MONEY`, `MISC_EVENT`, `MISC_AFF`, `PERCENT`, `MISC_ENT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/hebrewner_cc_300d_he_4.0.2_3.0_1660031325511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/hebrewner_cc_300d_he_4.0.2_3.0_1660031325511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("hebrewner_cc_300d", "he") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("""ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה. אבו-ג'וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.""") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he",) .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("hebrewner_cc_300d", "he") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val nlp_pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, ner, ner_converter)) val data = Seq("""ב- 25 לאוגוסט עצר השב"כ את מוחמד אבו-ג'וייד , אזרח ירדני , שגויס לארגון הפת"ח והופעל על ידי חיזבאללה. אבו-ג'וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.""").toDS.toDF("text") val result = nlp_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("he.ner").predict("""ח והופעל על ידי חיזבאללה. אבו-ג'וייד התכוון להקים חוליות טרור בגדה ובקרב ערביי ישראל , לבצע פיגוע ברכבת ישראל בנהריה , לפגוע במטרות ישראליות בירדן ולחטוף חיילים כדי לשחרר אסירים ביטחוניים.""") ```

## Results ```bash | | ner_chunk | entity_label | |---:|------------------:|---------------:| | 0 | 25 לאוגוסט | DATE | | 1 | השב"כ | ORG | | 2 | מוחמד אבו-ג'וייד | PERS | | 3 | ירדני | MISC_AFF | | 4 | הפת"ח | ORG | | 5 | חיזבאללה | ORG | | 6 | אבו-ג'וייד | PERS | | 7 | בגדה | LOC | | 8 | ערביי | MISC_AFF | | 9 | ישראל | LOC | | 10 | ברכבת ישראל | ORG | | 11 | בנהריה | LOC | | 12 | ישראליות | MISC_AFF | | 13 | בירדן < | LOC | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|hebrewner_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 4.0.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|he| |Size:|14.8 MB| ## References This model is trained on dataset obtained from [https://www.cs.bgu.ac.il/~elhadad/nlpproj/naama/](https://www.cs.bgu.ac.il/~elhadad/nlpproj/naama/) ## Benchmarking ```bash label tp fp fn prec rec f1 I-TIME 5 2 0 0.714286 1 0.833333 I-MISC_AFF 2 0 3 1 0.4 0.571429 B-MISC_EVENT 7 0 1 1 0.875 0.933333 B-LOC 180 24 37 0.882353 0.829493 0.855107 I-ORG 124 47 38 0.725146 0.765432 0.744745 B-DATE 50 4 7 0.925926 0.877193 0.900901 I-PERS 157 10 15 0.94012 0.912791 0.926254 I-DATE 39 7 8 0.847826 0.829787 0.83871 B-MISC_AFF 132 11 9 0.923077 0.93617 0.929577 I-MISC_EVENT 6 0 2 1 0.75 0.857143 B-TIME 4 0 1 1 0.8 0.888889 I-PERCENT 8 0 0 1 1 1 I-MISC_ENT 11 3 10 0.785714 0.52381 0.628571 B-MISC_ENT 8 1 5 0.888889 0.615385 0.727273 I-LOC 79 18 23 0.814433 0.77451 0.79397 B-PERS 231 22 26 0.913044 0.898833 0.905882 B-MONEY 36 2 2 0.947368 0.947368 0.947368 B-PERCENT 28 3 0 0.903226 1 0.949152 B-ORG 166 41 37 0.801932 0.817734 0.809756 I-MONEY 61 1 1 0.983871 0.983871 0.983871 Macro-average 1334 196 225 0.899861 0.826869 0.861822 Micro-average 1334 196 225 0.871895 0.855677 0.86371 ``` --- layout: model title: Detect details of cellular structures (biobert) author: John Snow Labs name: ner_cellular_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract different types of cells, protiens, and their sub-structures using pretrained NER model. ## Predicted Entities `protein`, `cell_type`, `RNA`, `DNA`, `cell_line` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CELLULAR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cellular_biobert_en_3.0.0_3.0_1617260803352.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cellular_biobert_en_3.0.0_3.0_1617260803352.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_cellular_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_cellular_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cellular.biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cellular_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| --- layout: model title: BioBERT Embeddings (Pubmed) author: John Snow Labs name: biobert_pubmed_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pubmed_base_cased_en_2.6.0_2.4_1598342186392.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pubmed_base_cased_en_2.6.0_2.4_1598342186392.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala val embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.pubmed_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_biobert_pubmed_base_cased_embeddings I [0.4227580428123474, -0.01985771767795086, -0.... hate [0.04862901195883751, 0.2535072863101959, -0.5... cancer [0.05491625890135765, 0.09395376592874527, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_pubmed_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: Pipeline to Detect Clinical Entities author: John Snow Labs name: ner_jsl_enriched_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_enriched_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_jsl_enriched_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_biobert_pipeline_en_3.4.1_3.0_1647869600266.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_biobert_pipeline_en_3.4.1_3.0_1647869600266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_jsl_enriched_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` ```scala val pipeline = new PretrainedPipeline("ner_jsl_enriched_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_enriched_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash +---------------------------+------------+ |chunks |entities | +---------------------------+------------+ |21-day-old |Age | |male |Gender | |mom |Gender | |she |Gender | |mild |Modifier | |problems with his breathing|Symptom_Name| |negative |Negation | |perioral cyanosis |Symptom_Name| |retractions |Symptom_Name| |mom |Gender | |Tylenol |Drug_Name | |His |Gender | |his |Gender | |respiratory congestion |Symptom_Name| |He |Gender | |tired |Symptom_Name| |fussy |Symptom_Name| |albuterol |Drug_Name | |His |Gender | |he |Gender | +---------------------------+------------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_enriched_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: English image_classifier_vit_flowers ViTForImageClassification from Sena author: John Snow Labs name: image_classifier_vit_flowers date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_flowers` is a English model originally trained by Sena. ## Predicted Entities `nergis`, `leylak`, `karanfil`, `zambak`, `menekse` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_flowers_en_4.1.0_3.0_1660166443375.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_flowers_en_4.1.0_3.0_1660166443375.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_flowers", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_flowers", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_flowers| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from EasthShin) author: John Snow Labs name: bert_qa_Klue_CommonSense_model date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Klue-CommonSense-model` is a English model orginally trained by `EasthShin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Klue_CommonSense_model_en_4.0.0_3.0_1654178775404.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Klue_CommonSense_model_en_4.0.0_3.0_1654178775404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Klue_CommonSense_model","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Klue_CommonSense_model","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.klue.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Klue_CommonSense_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|413.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/EasthShin/Klue-CommonSense-model - https://mindslab.ai:8080/kr/company - https://ainize.ai/EastHShin/Klue-CommonSense_QA?branch=main - https://ainize.ai/workspace/create?imageId=hnj95592adzr02xPTqss&git=https://github.com/EastHShin/Klue-CommonSense-workspace - https://main-klue-common-sense-qa-east-h-shin.endpoint.ainize.ai/ --- layout: model title: Smaller BERT Embeddings (L-2_H-128_A-2) author: John Snow Labs name: small_bert_L2_128 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L2_128_en_2.6.0_2.4_1598344320681.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L2_128_en_2.6.0_2.4_1598344320681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L2_128", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L2_128", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L2_128').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L2_128_embeddings I [-1.2788691520690918, -0.011364400386810303, 0.... love [-1.4087588787078857, -0.348095178604126, -0.... NLP [-1.6277656555175781, -0.28823617100715637, ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L2_128| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|128| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1](https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1) --- layout: model title: Legal Severability Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_severability_bert date: 2023-03-05 tags: [en, legal, classification, clauses, severability, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Severability` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Severability`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_severability_bert_en_1.0.0_3.0_1678050694994.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_severability_bert_en_1.0.0_3.0_1678050694994.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_severability_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Severability]| |[Other]| |[Other]| |[Severability]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_severability_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.98 1.00 0.99 263 Severability 1.00 0.98 0.99 233 accuracy - - 0.99 496 macro-avg 0.99 0.99 0.99 496 weighted-avg 0.99 0.99 0.99 496 ``` --- layout: model title: Relation Extraction Between Body Parts and Procedures author: John Snow Labs name: redl_bodypart_procedure_test_biobert date: 2021-09-10 tags: [relation_extraction, en, clinical, dl, licensed] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 2.4 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like ‘Internal_organ_or_component’, ’External_body_part_or_region’ etc. and procedure and test entities. `1` : body part and test/procedure are related to each other. `0` : body part and test/procedure are not related to each other. ## Predicted Entities `0`, `1` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_3.0.3_2.4_1631307197287.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_3.0.3_2.4_1631307197287.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_simple_kitchen", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_simple_kitchen", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_simple_kitchen| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Named Entity Recognition Profiling (Clinical) author: John Snow Labs name: ner_profiling_clinical date: 2022-08-30 tags: [en, clinical, profiling, ner_profiling, ner, licensed] task: [Named Entity Recognition, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `embeddings_clinical`. It has been updated by adding new clinical NER models and NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `jsl_ner_wip_clinical`, `jsl_ner_wip_greedy_clinical`, `jsl_ner_wip_modifier_clinical`, `jsl_rd_ner_wip_greedy_clinical`, `ner_abbreviation_clinical`, `ner_ade_binary`, `ner_ade_clinical`, `ner_anatomy`, `ner_anatomy_coarse`, `ner_bacterial_species`, `ner_biomarker`, `ner_biomedical_bc2gm`, `ner_bionlp`, `ner_cancer_genetics`, `ner_cellular`, `ner_chemd_clinical`, `ner_chemicals`, `ner_chemprot_clinical`, `ner_chexpert`, `ner_clinical`, `ner_clinical_large`, `ner_clinical_trials_abstracts`, `ner_covid_trials`, `ner_deid_augmented`, `ner_deid_enriched`, `ner_deid_generic_augmented`, `ner_deid_large`, `ner_deid_sd`, `ner_deid_sd_large`, `ner_deid_subentity_augmented`, `ner_deid_subentity_augmented_i2b2`, `ner_deid_synthetic`, `ner_deidentify_dl`, `ner_diseases`, `ner_diseases_large`, `ner_drugprot_clinical`, `ner_drugs`, `ner_drugs_greedy`, `ner_drugs_large`, `ner_events_admission_clinical`, `ner_events_clinical`, `ner_genetic_variants`, `ner_human_phenotype_gene_clinical`, `ner_human_phenotype_go_clinical`, `ner_jsl`, `ner_jsl_enriched`, `ner_jsl_greedy`, `ner_jsl_slim`, `ner_living_species`, `ner_measurements_clinical`, `ner_medmentions_coarse`, `ner_nature_nero_clinical`, `ner_nihss`, `ner_pathogen`, `ner_posology`, `ner_posology_experimental`, `ner_posology_greedy`, `ner_posology_large`, `ner_posology_small`, `ner_radiology`, `ner_radiology_wip_clinical`, `ner_risk_factors`, `ner_supplement_clinical`, `nerdl_tumour_demo` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.2.Pretrained_NER_Profiling_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.0.2_3.0_1661867359272.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.0.2_3.0_1661867359272.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline('ner_profiling_clinical', 'en', 'clinical/models') result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline("ner_profiling_clinical", "en", "clinical/models") val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_clinical").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ```

## Results ```bash ******************** ner_jsl Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('subsequent', 'Modifier'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Diabetes'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Communicable_Disease'), ('obesity', 'Obesity'), ('body mass index', 'Symptom'), ('33.5 kg/m2', 'Weight'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_diseases_large Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('vomiting', 'Disease')] ******************** ner_radiology Model Results ******************** [('gestational diabetes mellitus', 'Disease_Syndrome_Disorder'), ('type two diabetes mellitus', 'Disease_Syndrome_Disorder'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('acute hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Disease_Syndrome_Disorder'), ('body', 'BodyPart'), ('mass index', 'Symptom'), ('BMI', 'Test'), ('33.5', 'Measurements'), ('kg/m2', 'Units'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus', 'PROBLEM'), ('T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'PROBLEM'), ('BMI', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_medmentions_coarse Model Results ******************** [('female', 'Organism_Attribute'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('T2DM', 'Disease_or_Syndrome'), ('HTG-induced pancreatitis', 'Disease_or_Syndrome'), ('associated with', 'Qualitative_Concept'), ('acute hepatitis', 'Disease_or_Syndrome'), ('obesity', 'Disease_or_Syndrome'), ('body mass index', 'Clinical_Attribute'), ('BMI', 'Clinical_Attribute'), ('polyuria', 'Sign_or_Symptom'), ('polydipsia', 'Sign_or_Symptom'), ('poor appetite', 'Sign_or_Symptom'), ('vomiting', 'Sign_or_Symptom')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_clinical| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|2.6 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: Fast Neural Machine Translation Model from Tetela to English author: John Snow Labs name: opus_mt_tll_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tll, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tll` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tll_en_xx_2.7.0_2.4_1609168380675.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tll_en_xx_2.7.0_2.4_1609168380675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tll_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_huggingpics_package_demo_2 ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_huggingpics_package_demo_2 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_huggingpics_package_demo_2` is a English model originally trained by nateraw. ## Predicted Entities `corgi`, `husky`, `shibu inu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_huggingpics_package_demo_2_en_4.1.0_3.0_1660169461913.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_huggingpics_package_demo_2_en_4.1.0_3.0_1660169461913.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_huggingpics_package_demo_2", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_huggingpics_package_demo_2", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_huggingpics_package_demo_2| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Recognize Posology Pipeline author: John Snow Labs name: recognize_entities_posology date: 2021-03-29 tags: [ner, named_entity_recognition, pos, parts_of_speech, posology, ner_posology, pipeline, en, licensed] task: [Named Entity Recognition, Part of Speech Tagging] language: en nav_key: models edition: Spark NLP for Healthcare 3.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline detects drugs, dosage, form, frequency, duration, route, and drug strength in text. ## Predicted Entities `DRUG`, `STRENGTH`, `DURATION`, `FREQUENCY`, `FORM`, `DOSAGE`, `ROUTE`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/recognize_entities_posology_en_3.0.0_3.0_1617042229126.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/recognize_entities_posology_en_3.0.0_3.0_1617042229126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('recognize_entities_posology', 'en', 'clinical/models') annotations = pipeline.fullAnnotate("""The patient was perscriped 50MG penicilin for is headache""")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("recognize_entities_posology", "en", "clinical/models") val result = pipeline.fullAnnotate("""The patient was perscriped 50MG penicilin for is headache""")(0) ``` {:.nlu-block} ```python import nlu result_df = nlu.load('ner.posology').predict("""The patient was perscriped 50MG penicilin for is headache""") result_df ```

## Results ```bash +-----------------------------------------+ |result | +-----------------------------------------+ |[O, O, O, O, B-Strength, B-Drug, O, O, O]| +-----------------------------------------+ +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |ner | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |[[named_entity, 0, 2, O, [word -> The, confidence -> 1.0], []], [named_entity, 4, 10, O, [word -> patient, confidence -> 0.9993], []], [named_entity, 12, 14, O, [word -> was, confidence -> 1.0], []], [named_entity, 16, 25, O, [word -> perscriped, confidence -> 0.9985], []], [named_entity, 27, 30, B-Strength, [word -> 50MG, confidence -> 0.9966], []], [named_entity, 32, 40, B-Drug, [word -> penicilin, confidence -> 0.9934], []], [named_entity, 42, 44, O, [word -> for, confidence -> 0.9999], []], [named_entity, 46, 47, O, [word -> is, confidence -> 0.9468], []], [named_entity, 49, 56, O, [word -> headache, confidence -> 0.9805], []]]| +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|recognize_entities_posology| |Type:|pipeline| |Compatibility:|Spark NLP for Healthcare 3.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_500000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-500000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_500000_cased_generator_de_3.4.4_3.0_1652786423286.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_500000_cased_generator_de_3.4.4_3.0_1652786423286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_500000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|223.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-500000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: Financial NER (xl, Extra Large) author: John Snow Labs name: finner_financial_xlarge date: 2022-11-30 tags: [en, financial, ner, earning, calls, 10k, fillings, annual, reports, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a `xl` (extra-large) version of a financial model, trained in a combination of two data sets: Earning Calls and 10K Fillings. Please note this model requires some tokenization configuration to extract the currency (see python snippet below). The aim of this model is to detect the main pieces of financial information in annual reports of companies, more specifically this model is being trained with 10K filings. The currently available entities are: - AMOUNT: Numeric amounts, not percentages - ASSET: Current or Fixed Asset - ASSET_DECREASE: Decrease in the asset possession/exposure - ASSET_INCREASE: Increase in the asset possession/exposure - CF: Total cash flow - CF_DECREASE: Relative decrease in cash flow - CF_INCREASE: Relative increase in cash flow - COUNT: Number of items (not monetary, not percentages). - CURRENCY: The currency of the amount - DATE: Generic dates in context where either it's not a fiscal year or it can't be asserted as such given the context - EXPENSE: An expense or loss - EXPENSE_DECREASE: A piece of information saying there was an expense decrease in that fiscal year - EXPENSE_INCREASE: A piece of information saying there was an expense increase in that fiscal year - FCF: Free Cash Flow - FISCAL_YEAR: A date which expresses which month the fiscal exercise was closed for a specific year - KPI: Key Performance Indicator, a quantifiable measure of performance over time for a specific objective - KPI_DECREASE: Relative decrease in a KPI - KPI_INCREASE: Relative increase in a KPI - LIABILITY: Current or Long-Term Liability (not from stockholders) - LIABILITY_DECREASE: Relative decrease in liability - LIABILITY_INCREASE: Relative increase in liability - ORG: Mention to a company/organization name - PERCENTAGE: Numeric amounts which are percentages - PROFIT: Profit or also Revenue - PROFIT_DECLINE: A piece of information saying there was a profit / revenue decrease in that fiscal year - PROFIT_INCREASE: A piece of information saying there was a profit / revenue increase in that fiscal year - TICKER: Trading symbol of the company You can also check for the Relation Extraction model which connects these entities together ## Predicted Entities `AMOUNT`, `ASSET`, `ASSET_DECREASE`, `ASSET_INCREASE`, `CF`, `CF_DECREASE`, `CF_INCREASE`, `COUNT`, `CURRENCY`, `DATE`, `EXPENSE`, `EXPENSE_DECREASE`, `EXPENSE_INCREASE`, `FCF`, `FISCAL_YEAR`, `KPI`, `KPI_DECREASE`, `KPI_INCREASE`, `LIABILITY`, `LIABILITY_DECREASE`, `LIABILITY_INCREASE`, `ORG`, `PERCENTAGE`, `PROFIT`, `PROFIT_DECLINE`, `PROFIT_INCREASE`, `TICKER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_financial_xlarge_en_1.0.0_3.0_1669840074362.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_financial_xlarge_en_1.0.0_3.0_1669840074362.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_financial_xlarge", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""License fees revenue decreased 40 %, or 0.5 million to 0.7 million for the year ended December 31, 2020 compared to 1.2 million for the year ended December 31, 2019"""]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("text"), F.expr("cols['1']['entity']").alias("label")).show(200, truncate = False) ```

## Results ```bash +---------+----------------+----------+ | token| ner_label|confidence| +---------+----------------+----------+ | License|B-PROFIT_DECLINE| 0.9658| | fees|I-PROFIT_DECLINE| 0.7826| | revenue|I-PROFIT_DECLINE| 0.8992| |decreased| O| 1.0| | 40| B-PERCENTAGE| 0.9997| | %| O| 1.0| | ,| O| 0.9997| | or| O| 0.9999| | 0.5| B-AMOUNT| 0.9925| | million| I-AMOUNT| 0.9989| | to| O| 0.9996| | 0.7| B-AMOUNT| 0.9368| | million| I-AMOUNT| 0.9949| | for| O| 0.9999| | the| O| 0.9944| | year| O| 0.9976| | ended| O| 0.9987| | December| B-FISCAL_YEAR| 0.9941| | 31| I-FISCAL_YEAR| 0.8955| | ,| I-FISCAL_YEAR| 0.8869| | 2020| I-FISCAL_YEAR| 0.9941| | compared| O| 0.9999| | to| O| 0.9995| | 1.2| B-AMOUNT| 0.9853| | million| I-AMOUNT| 0.9831| | for| O| 0.9999| | the| O| 0.9914| | year| O| 0.9948| | ended| O| 0.9985| | December| B-FISCAL_YEAR| 0.9812| | 31| I-FISCAL_YEAR| 0.8185| | ,| I-FISCAL_YEAR| 0.8351| | 2019| I-FISCAL_YEAR| 0.9541| +---------+----------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_financial_xlarge| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References In-house annotations on Earning Calls and 10-K Filings combined. ## Benchmarking ```bash label tp fp fn prec rec f1 B-LIABILITY_INCREASE 1 0 0 1.0 1.0 1.0 I-AMOUNT 915 97 8 0.9041502 0.9913326 0.9457364 B-COUNT 7 1 2 0.875 0.7777778 0.8235294 I-LIABILITY_INCREASE 1 0 0 1.0 1.0 1.0 B-AMOUNT 1304 124 19 0.9131653 0.9856387 0.9480189 I-KPI 2 0 7 1.0 0.22222222 0.36363637 B-DATE 525 32 44 0.94254935 0.9226714 0.9325044 I-LIABILITY 156 49 97 0.7609756 0.6166008 0.68122274 I-DATE 343 12 36 0.9661972 0.9050132 0.93460494 B-CF_DECREASE 6 1 3 0.85714287 0.6666667 0.75 I-EXPENSE 270 86 74 0.75842696 0.78488374 0.77142864 I-KPI_INCREASE 0 0 1 0.0 0.0 0.0 B-LIABILITY 82 30 46 0.73214287 0.640625 0.6833333 I-CF 420 97 84 0.8123791 0.8333333 0.82272285 I-CF_DECREASE 17 3 12 0.85 0.5862069 0.6938776 I-COUNT 7 0 0 1.0 1.0 1.0 B-FCF 5 0 0 1.0 1.0 1.0 B-PROFIT_INCREASE 54 23 22 0.7012987 0.7105263 0.7058824 B-KPI_INCREASE 1 0 2 1.0 0.33333334 0.5 B-EXPENSE 118 42 36 0.7375 0.76623374 0.75159234 I-CF_INCREASE 43 0 17 1.0 0.71666664 0.8349514 I-PERCENTAGE 4 6 0 0.4 1.0 0.5714286 I-PROFIT_DECLINE 39 11 4 0.78 0.90697676 0.8387097 I-KPI_DECREASE 1 1 0 0.5 1.0 0.6666667 B-CF_INCREASE 23 0 2 1.0 0.92 0.9583333 I-PROFIT 228 118 19 0.6589595 0.9230769 0.7689713 B-CURRENCY 943 42 12 0.9573604 0.98743457 0.972165 I-PROFIT_INCREASE 80 34 16 0.7017544 0.8333333 0.7619047 B-CF 118 32 29 0.7866667 0.8027211 0.7946128 B-PROFIT 134 55 23 0.7089947 0.85350317 0.7745664 B-PERCENTAGE 281 17 7 0.942953 0.9756944 0.95904434 B-TICKER 2 0 0 1.0 1.0 1.0 I-FISCAL_YEAR 585 17 27 0.9717608 0.9558824 0.9637562 B-ORG 2 0 0 1.0 1.0 1.0 B-PROFIT_DECLINE 22 5 8 0.8148148 0.73333335 0.7719298 B-EXPENSE_INCREASE 35 7 4 0.8333333 0.8974359 0.86419755 B-EXPENSE_DECREASE 23 3 4 0.88461536 0.8518519 0.8679245 B-FISCAL_YEAR 195 6 12 0.9701493 0.942029 0.9558824 I-EXPENSE_DECREASE 46 9 16 0.8363636 0.7419355 0.78632486 I-FCF 10 0 0 1.0 1.0 1.0 I-EXPENSE_INCREASE 83 13 9 0.8645833 0.90217394 0.88297874 Macro-average 7134 977 728 0.77496254 0.72599226 0.74967855 Micro-average 7134 977 728 0.8795463 0.9074027 0.8932574 ``` --- layout: model title: Mapping Abbreviations and Acronyms of Medical Regulatory Activities with Their Definitions author: John Snow Labs name: abbreviation_mapper date: 2022-06-26 tags: [abbreviation, definition, licensed, clinical, en, chunk_mapper] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps abbreviations and acronyms of medical regulatory activities with their `definition`. ## Predicted Entities `definition` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/abbreviation_mapper_en_3.5.3_3.0_1656250645758.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/abbreviation_mapper_en_3.5.3_3.0_1656250645758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") abbr_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("abbr_ner") abbr_converter = NerConverter() \ .setInputCols(["sentence", "token", "abbr_ner"]) \ .setOutputCol("abbr_ner_chunk")\ chunkerMapper = ChunkMapperModel.pretrained("abbreviation_mapper", "en", "clinical/models")\ .setInputCols(["abbr_ner_chunk"])\ .setOutputCol("mappings")\ .setRels(["definition"])\ .setLowerCase(True) pipeline = Pipeline().setStages([ document_assembler, sentence_detector, tokenizer, word_embeddings, abbr_ner, abbr_converter, chunkerMapper]) sample_text = ["""Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet."""] data = spark.createDataFrame([sample_text]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val abbr_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("abbr_ner") val abbr_converter = new NerConverter() .setInputCols(Array("sentence", "token", "abbr_ner")) .setOutputCol("abbr_ner_chunk") val chunkerMapper = ChunkMapperModel.pretrained("abbreviation_mapper", "en", "clinical/models") .setInputCols("abbr_ner_chunk") .setOutputCol("mappings") .setRels(Array("definition")) .setLowerCase(True) val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, abbr_ner, abbr_converter, chunkerMapper)) val sample_text = """Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.""" val data = Seq(sample_text).toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.abbreviation_to_definition").predict("""Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.""") ```

## Results ```bash +-----+----------------------------+ |chunk|mapping_result | +-----+----------------------------+ |CBC |complete blood count | |HIV |human immunodeficiency virus| +-----+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|abbreviation_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[abbr_ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|219.9 KB| ## References https://www.johnsnowlabs.com/marketplace/list-of-abbreviations-and-acronyms-for-medical-regulatory-activities/ --- layout: model title: English DistilBertForQuestionAnswering Cased model (from kevinbram) author: John Snow Labs name: distilbert_qa_testarbara date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `testarbara` is a English model originally trained by `kevinbram`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_testarbara_en_4.3.0_3.0_1672775789186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_testarbara_en_4.3.0_3.0_1672775789186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_testarbara","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_testarbara","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_testarbara| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/kevinbram/testarbara --- layout: model title: English asr_wav2vec2_base_100h_by_facebook TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_base_100h_by_facebook date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_100h_by_facebook` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_100h_by_facebook_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_by_facebook_en_4.2.0_3.0_1664038653083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_100h_by_facebook_en_4.2.0_3.0_1664038653083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_100h_by_facebook", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_100h_by_facebook", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_100h_by_facebook| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.9 MB| --- layout: model title: English RobertaForQuestionAnswering (from LucasS) author: John Snow Labs name: roberta_qa_robertaABSA date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `robertaABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaABSA_en_4.0.0_3.0_1655738674279.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_robertaABSA_en_4.0.0_3.0_1655738674279.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_robertaABSA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_robertaABSA","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta_absa").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_robertaABSA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|436.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LucasS/robertaABSA --- layout: model title: Chinese BertForMaskedLM Large Cased model (from hfl) author: John Snow Labs name: bert_embeddings_chinese_mac_large date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-macbert-large` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_mac_large_zh_4.2.4_3.0_1670021258806.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_mac_large_zh_4.2.4_3.0_1670021258806.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_mac_large","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_mac_large","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_mac_large| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-macbert-large - https://github.com/ymcui/MacBERT/blob/master/LICENSE - https://2020.emnlp.org - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://github.com/chatopera/Synonyms - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/2004.13922 --- layout: model title: English asr_wav2vec2_xls_r_300m TFWav2Vec2ForCTC from hgharibi author: John Snow Labs name: pipeline_asr_wav2vec2_xls_r_300m date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m` is a English model originally trained by hgharibi. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xls_r_300m_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_en_4.2.0_3.0_1664016873292.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xls_r_300m_en_4.2.0_3.0_1664016873292.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xls_r_300m', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xls_r_300m", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xls_r_300m| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal General Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_general_bert date: 2023-03-05 tags: [en, legal, classification, clauses, general, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `General` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `General`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_general_bert_en_1.0.0_3.0_1678049996624.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_general_bert_en_1.0.0_3.0_1678049996624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_general_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[General]| |[Other]| |[Other]| |[General]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_general_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support General 0.77 0.81 0.79 122 Other 0.84 0.81 0.82 150 accuracy - - 0.81 272 macro-avg 0.81 0.81 0.81 272 weighted-avg 0.81 0.81 0.81 272 ``` --- layout: model title: Legal Credit agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_credit_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_credit_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class credit-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `credit-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_credit_agreement_en_1.0.0_3.0_1666620886293.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_credit_agreement_en_1.0.0_3.0_1666620886293.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_credit_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[credit-agreement]| |[other]| |[other]| |[credit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_credit_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support credit-agreement 0.94 0.94 0.94 36 other 0.97 0.97 0.97 62 accuracy - - 0.96 98 macro-avg 0.96 0.96 0.96 98 weighted-avg 0.96 0.96 0.96 98 ``` --- layout: model title: Pipeline to Mapping RxNORM Codes with Their Corresponding UMLS Codes author: John Snow Labs name: rxnorm_umls_mapping date: 2022-06-27 tags: [rxnorm, umls, pipeline, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `rxnorm_umls_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_3.5.3_3.0_1656367260714.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapping_en_3.5.3_3.0_1656367260714.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rxnorm_umls_mapping", "en", "clinical/models") result= pipeline.fullAnnotate("1161611 315677") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rxnorm_umls_mapping", "en", "clinical/models") val result= pipeline.fullAnnotate("1161611 315677") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.umls").predict("""1161611 315677""") ```

## Results ```bash | | rxnorm_code | umls_code | |---:|:-----------------|:--------------------| | 0 | 1161611 | 315677 | C3215948 | C0984912 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.9 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_2_h_768 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-2_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_2_h_768_zh_4.2.4_3.0_1670325904665.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_2_h_768_zh_4.2.4_3.0_1670325904665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_2_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_2_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_2_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|117.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-2_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Relation Extraction between Biomarkers and Results (ReDL) author: John Snow Labs name: redl_oncology_biomarker_result_biobert_wip date: 2022-09-29 tags: [licensed, clinical, oncology, en, relation_extraction, test, biomarker] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links Biomarker and Oncogene extractions to their corresponding Biomarker_Result extractions. ## Predicted Entities `is_finding_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biomarker_result_biobert_wip_en_4.1.0_3.0_1664457944095.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biomarker_result_biobert_wip_en_4.1.0_3.0_1664457944095.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash | chunk1 | entity1 | chunk2 | entity2 | relation | confidence | |----------|------------------|--------------------------------|------------------|---------------|------------| | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | 0.99808085 | | negative | Biomarker_Result | napsin | Biomarker | is_finding_of | 0.99637383 | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | 0.99221414 | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | 0.9893672 | | positive | Biomarker_Result | HER2 | Oncogene | O | 0.9986272 | | ER | Biomarker | negative | Biomarker_Result | O | 0.9999089 | | PR | Biomarker | negative | Biomarker_Result | O | 0.9998932 | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | 0.98810333 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_biomarker_result_biobert_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.93 0.97 0.95 is_finding_of 0.97 0.93 0.95 macro-avg 0.95 0.95 0.95 ``` --- layout: model title: English asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6 TFWav2Vec2ForCTC from chrisvinsen author: John Snow Labs name: pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6` is a English model originally trained by chrisvinsen. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_en_4.2.0_3.0_1664107389148.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6_en_4.2.0_3.0_1664107389148.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xlsr_wav2vec2_base_commonvoice_demo_colab_6| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Portuguese BertForQuestionAnswering model (from pierreguillou) author: John Snow Labs name: bert_qa_bert_large_cased_squad_v1.1_portuguese date: 2022-06-06 tags: [pt, open_source, question_answering, bert] task: Question Answering language: pt edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-cased-squad-v1.1-portuguese` is a Portuguese model orginally trained by `pierreguillou`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_cased_squad_v1.1_portuguese_pt_4.0.0_3.0_1654536169488.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_cased_squad_v1.1_portuguese_pt_4.0.0_3.0_1654536169488.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_cased_squad_v1.1_portuguese","pt") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_cased_squad_v1.1_portuguese","pt") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("pt.answer_question.squad.bert.large_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_cased_squad_v1.1_portuguese| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|pt| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/pierreguillou/bert-large-cased-squad-v1.1-portuguese - https://github.com/piegu/language-models/blob/master/question_answering_BERT_large_cased_squad_v11_pt.ipynb - https://nbviewer.jupyter.org/github/piegu/language-models/blob/master/question_answering_BERT_large_cased_squad_v11_pt.ipynb - https://medium.com/@pierre_guillou/nlp-como-treinar-um-modelo-de-question-answering-em-qualquer-linguagem-baseado-no-bert-large-1c899262dd96#c2f5 - https://ailab.unb.br/ - https://www.linkedin.com/in/pierreguillou/ - http://www.deeplearningbrasil.com.br/ - https://neuralmind.ai/ - https://medium.com/@pierre_guillou/nlp-como-treinar-um-modelo-de-question-answering-em-qualquer-linguagem-baseado-no-bert-large-1c899262dd96 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_base_finetuned_deletion_squad_15 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-deletion-squad-15` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_deletion_squad_15_en_4.3.0_3.0_1674216599443.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_deletion_squad_15_en_4.3.0_3.0_1674216599443.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_deletion_squad_15","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_deletion_squad_15","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_deletion_squad_15| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-deletion-squad-15 --- layout: model title: ICD10CM Injuries Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_injuries_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-28 task: Entity Resolution edition: Healthcare NLP 2.4.5 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_injuries_clinical_en_2.4.5_2.4_1588103825347.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_injuries_clinical_en_2.4.5_2.4_1588103825347.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... injury_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_injuries_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, injury_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val injury_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_injuries_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, injury_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash chunk entity icd10_inj_description icd10_inj_code 0 a cold, cough PROBLEM Insect bite (nonvenomous) of throat, sequela S1016XS 1 runny nose PROBLEM Abrasion of nose, initial encounter S0031XA 2 fever PROBLEM Blister (nonthermal) of abdominal wall, initia... S30821A 3 difficulty breathing PROBLEM Concussion without loss of consciousness, init... S060X0A 4 her cough PROBLEM Contusion of throat, subsequent encounter S100XXD 5 physical exam TEST Concussion without loss of consciousness, init... S060X0A 6 fairly congested PROBLEM Contusion and laceration of right cerebrum wit... S06310A 7 Amoxil TREATMENT Insect bite (nonvenomous), unspecified ankle, ... S90569S 8 Aldex TREATMENT Insect bite (nonvenomous) of penis, initial en... S30862A 9 difficulty breathing PROBLEM Concussion without loss of consciousness, init... S060X0A 10 more congested PROBLEM Complete traumatic amputation of two or more r... S98211S 11 trouble sleeping PROBLEM Concussion without loss of consciousness, sequela S060X0S 12 congestion PROBLEM Unspecified injury of portal vein, initial enc... S35319A ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------------------| | Name: | chunkresolve_icd10cm_injuries_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD10CM Dataset Range: S0000XA-S98929S https://www.icd10data.com/ICD10CM/Codes/S00-T88 --- layout: model title: English image_classifier_vit_pneumonia_test_attempt ViTForImageClassification from eren23 author: John Snow Labs name: image_classifier_vit_pneumonia_test_attempt date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pneumonia_test_attempt` is a English model originally trained by eren23. ## Predicted Entities `NORMAL`, `PNEUMONIA` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pneumonia_test_attempt_en_4.1.0_3.0_1660167722454.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pneumonia_test_attempt_en_4.1.0_3.0_1660167722454.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pneumonia_test_attempt", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pneumonia_test_attempt", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pneumonia_test_attempt| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Detect Living Species(bert_base_cased) author: John Snow Labs name: ner_living_species_bert date: 2022-06-22 tags: [es, ner, clinical, licensed, bert] task: Named Entity Recognition language: es edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Spanish which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `bert_base_cased` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_es_3.5.3_3.0_1655906269739.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_es_3.5.3_3.0_1655906269739.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_base_cased", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "es","clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_base_cased", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "es","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.living_species.bert").predict("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""") ```

## Results ```bash +--------------+-------+ |ner_chunk |label | +--------------+-------+ |Lactante varón|HUMAN | |familiares |HUMAN | |personales |HUMAN | |neonatal |HUMAN | |legumbres |SPECIES| |lentejas |SPECIES| |garbanzos |SPECIES| |legumbres |SPECIES| |madre |HUMAN | |Cacahuete |SPECIES| |padres |HUMAN | +--------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|16.4 MB| ## References [https://temu.bsc.es/livingner/](https://temu.bsc.es/livingner/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 1.00 1.00 1.00 3281 B-SPECIES 1.00 1.00 1.00 3712 I-HUMAN 1.00 0.99 0.99 297 I-SPECIES 0.90 0.99 0.95 1732 micro-avg 0.98 1.00 0.99 9022 macro-avg 0.97 0.99 0.98 9022 weighted-avg 0.98 1.00 0.99 9022 ``` --- layout: model title: Legal Limited Liability Company Operating Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_limited_liability_company_operating_agreement_bert date: 2023-02-02 tags: [en, legal, classification, limited, liability, company, operating, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_limited_liability_company_operating_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `limited-liability-company-operating-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `limited-liability-company-operating-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_limited_liability_company_operating_agreement_bert_en_1.0.0_3.0_1675360839682.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_limited_liability_company_operating_agreement_bert_en_1.0.0_3.0_1675360839682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_limited_liability_company_operating_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[limited-liability-company-operating-agreement]| |[other]| |[other]| |[limited-liability-company-operating-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_limited_liability_company_operating_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support limited-liability-company-operating-agreement 1.00 0.94 0.97 53 other 0.98 1.00 0.99 122 accuracy - - 0.98 175 macro-avg 0.99 0.97 0.98 175 weighted-avg 0.98 0.98 0.98 175 ``` --- layout: model title: Pipeline to find clinical events and find temporal relations (ERA) author: John Snow Labs name: explain_clinical_doc_era date: 2023-04-20 tags: [pipeline, en, licensed, clinical] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline with `ner_clinical_events`, `assertion_dl` and `re_temporal_events_clinical`. It will extract clinical entities, assign assertion status and find temporal relationships between clinical entities. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_4.3.0_3.2_1682022511508.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_4.3.0_3.2_1682022511508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("explain_clinical_doc_era", "en", "clinical/models") text = """She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("explain_clinical_doc_era", "en", "clinical/models") val text = """She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """ val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.era").predict("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """) ```

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|:-----------|:--------------|----------------:|--------------:|:--------------------------|:--------------|----------------:|--------------:|:------------------------------|-------------:| | 0 | AFTER | OCCURRENCE | 7 | 14 | admitted | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | 0.963836 | | 1 | BEFORE | OCCURRENCE | 7 | 14 | admitted | DATE | 45 | 54 | 2 days ago | 0.587098 | | 2 | BEFORE | OCCURRENCE | 7 | 14 | admitted | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.999991 | | 3 | OVERLAP | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | DATE | 45 | 54 | 2 days ago | 0.996056 | | 4 | BEFORE | CLINICAL_DEPT | 19 | 43 | The John Hopkins Hospital | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.995216 | | 5 | OVERLAP | DATE | 45 | 54 | 2 days ago | PROBLEM | 74 | 102 | gestational diabetes mellitus | 0.996954 | | 6 | BEFORE | EVIDENTIAL | 119 | 124 | denied | PROBLEM | 126 | 129 | pain | 1 | | 7 | BEFORE | EVIDENTIAL | 119 | 124 | denied | PROBLEM | 135 | 146 | any headache | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_era| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - DependencyParserModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - RelationExtractionModel - AssertionDLModel --- layout: model title: Part of Speech for Greek author: John Snow Labs name: pos_ud_gdt date: 2021-03-08 tags: [part_of_speech, open_source, greek, pos_ud_gdt, el] task: Part of Speech Tagging language: el edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - DET - X - VERB - ADP - NUM - NOUN - ADV - PUNCT - CCONJ - ADJ - AUX - PROPN - PRON - SCONJ - PART - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gdt_el_3.0.0_3.0_1615230364351.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gdt_el_3.0.0_3.0_1615230364351.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_gdt", "el") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Γεια σας από το John Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_gdt", "el") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Γεια σας από το John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Γεια σας από το John Snow Labs! ""] token_df = nlu.load('el.pos.ud_gdt').predict(text) token_df ```

## Results ```bash token pos 0 Γεια NOUN 1 σας PRON 2 από ADP 3 το DET 4 John X 5 Snow X 6 Labs X 7 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gdt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|el| --- layout: model title: Pipeline to Detect Living Species(embeddings_scielo_300d) author: John Snow Labs name: ner_living_species_300_pipeline date: 2023-03-09 tags: [licensed, clinical, es, ner] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species_300](https://nlp.johnsnowlabs.com/2022/11/22/ner_living_species_300_es.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_pipeline_es_4.3.0_3.2_1678392015205.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_pipeline_es_4.3.0_3.2_1678392015205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_300_pipeline", "es", "clinical/models") text = '''Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_300_pipeline", "es", "clinical/models") val text = "Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:---------------|--------:|------:|:------------|-------------:| | 0 | Lactante varón | 0 | 13 | HUMAN | 0.92045 | | 1 | familiares | 41 | 50 | HUMAN | 1 | | 2 | personales | 78 | 87 | HUMAN | 1 | | 3 | neonatal | 116 | 123 | HUMAN | 0.9817 | | 4 | legumbres | 162 | 170 | SPECIES | 0.9972 | | 5 | lentejas | 243 | 250 | SPECIES | 0.9592 | | 6 | garbanzos | 254 | 262 | SPECIES | 0.9754 | | 7 | legumbres | 290 | 298 | SPECIES | 0.9975 | | 8 | madre | 334 | 338 | HUMAN | 1 | | 9 | Cacahuete | 616 | 624 | SPECIES | 0.9963 | | 10 | padres | 728 | 733 | HUMAN | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_300_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|230.4 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English BertForQuestionAnswering model (from Laikokwei) author: John Snow Labs name: bert_qa_Laikokwei_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `Laikokwei`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Laikokwei_bert_finetuned_squad_en_4.0.0_3.0_1654535522132.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Laikokwei_bert_finetuned_squad_en_4.0.0_3.0_1654535522132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Laikokwei_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Laikokwei_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_Laikokwei").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Laikokwei_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Laikokwei/bert-finetuned-squad --- layout: model title: Named Entity Recognition Profiling (Biobert) author: John Snow Labs name: ner_profiling_biobert date: 2023-06-13 tags: [licensed, en, clinical, biobert, profiling, ner_profiling, ner] task: [Named Entity Recognition, Pipeline Healthcare] language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `biobert_pubmed_base_cased`. It has been updated by adding NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `jsl_ner_wip_greedy_biobert`, `jsl_rd_ner_wip_greedy_biobert`, `ner_ade_biobert`, `ner_anatomy_biobert`, `ner_anatomy_coarse_biobert`, `ner_bionlp_biobert`, `ner_cellular_biobert`, `ner_chemprot_biobert`, `ner_clinical_biobert`, `ner_deid_biobert`, `ner_deid_enriched_biobert`, `ner_diseases_biobert`, `ner_events_biobert`, `ner_human_phenotype_gene_biobert`, `ner_human_phenotype_go_biobert`, `ner_jsl_biobert`, `ner_jsl_enriched_biobert`, `ner_jsl_greedy_biobert`, `ner_living_species_biobert`, `ner_posology_biobert`, `ner_posology_large_biobert`, `ner_risk_factors_biobert` ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_4.4.4_3.2_1686663346965.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_4.4.4_3.2_1686663346965.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline('ner_profiling_biobert', 'en', 'clinical/models') result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline("ner_profiling_biobert", "en", "clinical/models") val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_biobert").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ```

## Results ```bash Results ******************** ner_diseases_biobert Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('type two diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('HTG-induced pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('poor appetite', 'Disease'), ('vomiting', 'Disease')] ******************** ner_events_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('eight years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('three years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'TEST'), ('BMI', 'TEST'), ('presented', 'OCCURRENCE'), ('a one-week', 'DURATION'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_jsl_biobert Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Obesity'), ('body mass index', 'BMI'), ('BMI ) of 33.5 kg/m2', 'BMI'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index ( BMI )', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_risk_factors_biobert Model Results ******************** [('diabetes mellitus', 'DIABETES'), ('subsequent type two diabetes mellitus', 'DIABETES'), ('obesity', 'OBESE')] {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_biobert| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|766.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: Czech asr_wav2vec2_xls_r_300m_250 TFWav2Vec2ForCTC from comodoro author: John Snow Labs name: asr_wav2vec2_xls_r_300m_250 date: 2022-09-25 tags: [wav2vec2, cs, audio, open_source, asr] task: Automatic Speech Recognition language: cs edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_250` is a Czech model originally trained by comodoro. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_250_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_250_cs_4.2.0_3.0_1664119316377.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_250_cs_4.2.0_3.0_1664119316377.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_250", "cs")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_250", "cs") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_250| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|cs| |Size:|1.2 GB| --- layout: model title: Sarcasm Classifier author: John Snow Labs name: classifierdl_use_sarcasm date: 2021-01-09 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [open_source, en, classifier] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify if a text contains sarcasm. ## Predicted Entities `normal`, `sarcasm` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_SARCASM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN_SARCASM.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_sarcasm_en_2.7.1_2.4_1610210956231.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_sarcasm_en_2.7.1_2.4_1610210956231.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained(lang="en") \ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") document_classifier = ClassifierDLModel.pretrained('classifierdl_use_sarcasm', 'en') \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") nlpPipeline = Pipeline(stages=[documentAssembler, use, document_classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate('If I could put into words how much I love waking up at am on Tuesdays I would') ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val use = UniversalSentenceEncoder.pretrained(lang="en") .setInputCols(Array("document")) .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_use_sarcasm", "en") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, use, document_classifier)) val data = Seq("If I could put into words how much I love waking up at am on Tuesdays I would").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""If I could put into words how much I love waking up at am on Tuesdays I would"""] sarcasm_df = nlu.load('classify.sarcasm.use').predict(text, output_level='document') sarcasm_df[["document", "sarcasm"]] ```

## Results ```bash +--------------------------------------------------------------------------------------------------------+------------+ |document |class | +--------------------------------------------------------------------------------------------------------+------------+ |If I could put into words how much I love waking up at am on Tuesdays I would | sarcasm | +--------------------------------------------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_sarcasm| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|tfhub_use| ## Data Source http://www.cs.utah.edu/~riloff/pdfs/official-emnlp13-sarcasm.pdf ## Benchmarking ```bash precision recall f1-score support normal 0.98 0.89 0.93 495 sarcasm 0.60 0.91 0.73 93 accuracy 0.89 588 macro avg 0.79 0.90 0.83 588 weighted avg 0.92 0.89 0.90 588 ``` --- layout: model title: Spanish Bert Embeddings (Base, Pasage, Allqa) author: John Snow Labs name: bert_embeddings_dpr_spanish_passage_encoder_allqa_base date: 2022-04-11 tags: [bert, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `dpr-spanish-passage_encoder-allqa-base` is a Spanish model orginally trained by `IIC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_dpr_spanish_passage_encoder_allqa_base_es_3.4.2_3.0_1649671207246.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_dpr_spanish_passage_encoder_allqa_base_es_3.4.2_3.0_1649671207246.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_dpr_spanish_passage_encoder_allqa_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|412.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/IIC/dpr-spanish-passage_encoder-allqa-base - https://arxiv.org/abs/2004.04906 - https://github.com/facebookresearch/DPR - https://arxiv.org/abs/2004.04906 - https://paperswithcode.com/sota?task=text+similarity&dataset=squad_es --- layout: model title: English image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1 ViTForImageClassification from AykeeSalazar author: John Snow Labs name: image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1` is a English model originally trained by AykeeSalazar. ## Predicted Entities `nonViolation`, `publicDrinking`, `publicSmoking` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1_en_4.1.0_3.0_1660170865642.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1_en_4.1.0_3.0_1660170865642.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vc_bantai__withoutAMBI_adunest_v1| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect Chemicals in Medical text (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_chemicals_pipeline date: 2023-03-20 tags: [berfortokenclassification, ner, chemicals, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemicals](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemicals_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_4.3.0_3.2_1679306458020.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_4.3.0_3.2_1679306458020.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") text = '''The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") val text = "The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemicals_pipeline").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------------|--------:|------:|:------------|-------------:| | 0 | p - choloroaniline | 40 | 57 | CHEM | 0.999986 | | 1 | chlorhexidine - digluconate | 90 | 116 | CHEM | 0.999989 | | 2 | kanamycin | 169 | 177 | CHEM | 0.999985 | | 3 | colistin | 181 | 188 | CHEM | 0.999982 | | 4 | povidone - iodine | 194 | 210 | CHEM | 0.99998 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Translate English to Tigrinya Pipeline author: John Snow Labs name: translate_en_ti date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ti, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ti` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ti_xx_2.7.0_2.4_1609700767749.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ti_xx_2.7.0_2.4_1609700767749.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ti", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ti", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ti').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ti| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Conflict of interest Clause Binary Classifier author: John Snow Labs name: legclf_conflict_of_interest_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `conflict-of-interest` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `conflict-of-interest` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_conflict_of_interest_clause_en_1.0.0_3.2_1660122277665.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_conflict_of_interest_clause_en_1.0.0_3.2_1660122277665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_conflict_of_interest_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[conflict-of-interest]| |[other]| |[other]| |[conflict-of-interest]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_conflict_of_interest_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support conflict-of-interest 0.90 0.81 0.85 32 other 0.95 0.97 0.96 108 accuracy - - 0.94 140 macro-avg 0.92 0.89 0.91 140 weighted-avg 0.93 0.94 0.93 140 ``` --- layout: model title: English RobertaForSequenceClassification Cased model (from masifayub) author: John Snow Labs name: roberta_classifier_autotrain_pan_976832386 date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification, tensorflow] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-PAN-976832386` is a English model originally trained by `masifayub`. ## Predicted Entities `1`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_pan_976832386_en_4.2.4_3.0_1670623208389.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_autotrain_pan_976832386_en_4.2.4_3.0_1670623208389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_pan_976832386","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_autotrain_pan_976832386","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_autotrain_pan_976832386| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|309.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/masifayub/autotrain-PAN-976832386 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0_en_4.3.0_3.0_1674213299788.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0_en_4.3.0_3.0_1674213299788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_1024_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|439.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-0 --- layout: model title: Medical Question Answering (biogpt_pubmed_qa) author: John Snow Labs name: biogpt_pubmed_qa date: 2023-05-15 tags: [licensed, clinical, en, biogpt, pubmed, qa, question_answering, tensorflow] task: Question Answering language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model has been trained with medical documents and can generate two types of answers, short and long. Types of questions are supported: `"short"` (producing yes/no/maybe) answers and `"full"` (long answers). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/biogpt_pubmed_qa_en_4.4.2_3.0_1684165576397.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/biogpt_pubmed_qa_en_4.4.2_3.0_1684165576397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------+ |result | +--------------+ |[sidd, besser]| +--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|lb| |Size:|1.9 KB| --- layout: model title: Translate Azerbaijani to English Pipeline author: John Snow Labs name: translate_az_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, az, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `az` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_az_en_xx_2.7.0_2.4_1609686225262.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_az_en_xx_2.7.0_2.4_1609686225262.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_az_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_az_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.az.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_az_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Trade Document Classifier (EURLEX) author: John Snow Labs name: legclf_trade_bert date: 2023-03-06 tags: [en, legal, classification, clauses, trade, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_trade_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Trade or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Trade`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_trade_bert_en_1.0.0_3.0_1678111687840.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_trade_bert_en_1.0.0_3.0_1678111687840.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_trade_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Trade]| |[Other]| |[Other]| |[Trade]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_trade_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.0 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.83 0.87 0.85 778 Trade 0.89 0.86 0.88 995 accuracy - - 0.86 1773 macro-avg 0.86 0.86 0.86 1773 weighted-avg 0.86 0.86 0.86 1773 ``` --- layout: model title: Sundanese RoBERTa Embeddings (from w11wo) author: John Snow Labs name: roberta_embeddings_sundanese_roberta_base date: 2022-04-14 tags: [roberta, embeddings, su, open_source] task: Embeddings language: su edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `sundanese-roberta-base` is a Sundanese model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_sundanese_roberta_base_su_3.4.2_3.0_1649948770581.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_sundanese_roberta_base_su_3.4.2_3.0_1649948770581.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_sundanese_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|su| |Size:|468.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/sundanese-roberta-base - https://arxiv.org/abs/1907.11692 - https://hf.co/datasets/oscar - https://hf.co/datasets/mc4 - https://hf.co/datasets/cc100 - https://su.wikipedia.org/ - https://hf.co/w11wo/sundanese-roberta-base/tree/main - https://hf.co/w11wo/sundanese-roberta-base/tensorboard - https://w11wo.github.io/ --- layout: model title: English asr_wav2vec2_base_timit_demo_test_jong TFWav2Vec2ForCTC from prows12 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_test_jong date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_test_jong` is a English model originally trained by prows12. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_test_jong_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_test_jong_en_4.2.0_3.0_1664101569048.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_test_jong_en_4.2.0_3.0_1664101569048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_test_jong', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_test_jong", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_test_jong| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Kaonde to English Pipeline author: John Snow Labs name: translate_kqn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, kqn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `kqn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_kqn_en_xx_2.7.0_2.4_1609691630870.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_kqn_en_xx_2.7.0_2.4_1609691630870.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_kqn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_kqn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.kqn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_kqn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Burmese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, my, open_source] task: Embeddings language: my edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_my_3.4.1_3.0_1647288529217.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_my_3.4.1_3.0_1647288529217.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|my| |Size:|195.6 MB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Legal Brokers Clause Binary Classifier author: John Snow Labs name: legclf_brokers_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `brokers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `brokers` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_brokers_clause_en_1.0.0_3.2_1660122187570.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_brokers_clause_en_1.0.0_3.2_1660122187570.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_brokers_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[brokers]| |[other]| |[other]| |[brokers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_brokers_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support brokers 0.96 0.99 0.97 77 other 1.00 0.99 0.99 219 accuracy - - 0.99 296 macro-avg 0.98 0.99 0.98 296 weighted-avg 0.99 0.99 0.99 296 ``` --- layout: model title: Sentence Entity Resolver for ICD-10-CM Codes (sbertresolve_icd10cm_augmented) author: John Snow Labs name: sbertresolve_icd10cm_augmented date: 2023-05-31 tags: [en, clinical, licensed, icd10cm, entity_resolution] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to ICD-10-CM codes using `sbert_jsl_medium_uncased` sentence bert embeddings. It also returns the official resolution text within the brackets inside the metadata. The model is augmented with synonyms, and previous augmentations are flexed according to cosine distances to unnormalized terms (ground truths). ## Predicted Entities `ICD-10-CM Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_augmented_en_4.4.2_3.0_1685531969657.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbertresolve_icd10cm_augmented_en_4.4.2_3.0_1685531969657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | ner_chunk| entity|icd10_code| resolutions| all_codes| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.4|[gestational diabetes mellitus [gestational diabetes mellitus], maternal...| [O24.4, O24.41, O24.43, Z86.32, K86.8, P70.2, O24.434, E10.9, O24.430]| |subsequent type two diabetes mellitus|PROBLEM| E11|[type 2 diabetes mellitus [type 2 diabetes mellitus], type ii diabetes m...|[E11, E11.9, E10.9, E10, E13.9, Z83.3, L83, E11.8, E11.32, E10.8, Z86.39...| | acute hepatitis|PROBLEM| K72.0|[acute hepatitis [acute and subacute hepatic failure], acute hepatitis a...|[K72.0, B15, B17.2, B17.1, B16, B17.9, B18.8, B15.9, K75.2, K73.9, B17.1...| | obesity|PROBLEM| E66.9|[obesity [obesity, unspecified], upper body obesity [other obesity], chi...| [E66.9, E66.8, P90, Q13.0, M79.4, Z86.39]| | a body mass index|PROBLEM| E66.9|[observation of body mass index [obesity, unspecified], finding of body ...|[E66.9, Z68.41, Z68, E66.8, Z68.45, Z68.4, Z68.1, Z68.2, R22.9, Z68.22, ...| | polyuria|PROBLEM| R35|[polyuria [polyuria], sialuria [other specified metabolic disorders], st...|[R35, E88.8, R30.0, N28.89, O04.8, R82.4, E74.8, R82.2, E73.9, R82.0, R3...| | polydipsia|PROBLEM| R63.1|[polydipsia [polydipsia], polyotia [accessory auricle], polysomia [conjo...|[R63.1, Q17.0, Q89.4, Q89.09, Q74.8, H53.8, H53.2, Q13.2, R63.8, E23.2, ...| | poor appetite|PROBLEM| R63.0|[poor appetite [anorexia], excessive appetite [polyphagia], poor feeding...|[R63.0, R63.2, P92.9, R45.81, Z55.8, R41.84, R41.3, Z74.8, R46.89, R45.8...| | vomiting|PROBLEM| R11.1|[vomiting [vomiting], vomiting bile [vomiting following gastrointestinal...|[R11.1, K91.0, K92.0, A08.39, R11, P92.0, P92.09, R11.12, R11.10, O21.9,...| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[bert_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|938.1 MB| |Case sensitive:|false| --- layout: model title: Translate English to Ewe Pipeline author: John Snow Labs name: translate_en_ee date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ee, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ee` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ee_xx_2.7.0_2.4_1609686384150.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ee_xx_2.7.0_2.4_1609686384150.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ee", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ee", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ee').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ee| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265909` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909_en_4.0.0_3.0_1655985665052.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909_en_4.0.0_3.0_1655985665052.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265909").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265909| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265909 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from allenai) author: John Snow Labs name: t5_unifiedqa_v2_small_1251000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unifiedqa-v2-t5-small-1251000` is a English model originally trained by `allenai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_unifiedqa_v2_small_1251000_en_4.3.0_3.0_1675158019272.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_unifiedqa_v2_small_1251000_en_4.3.0_3.0_1675158019272.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_unifiedqa_v2_small_1251000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_unifiedqa_v2_small_1251000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_unifiedqa_v2_small_1251000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|148.1 MB| ## References - https://huggingface.co/allenai/unifiedqa-v2-t5-small-1251000 - #further-details-httpsgithubcomallenaiunifiedqa - https://github.com/allenai/unifiedqa - #further-details-httpsgithubcomallenaiunifiedqa - https://github.com/allenai/unifiedqa --- layout: model title: English image_classifier_vit_pond_image_classification_6 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_pond_image_classification_6 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pond_image_classification_6` is a English model originally trained by SummerChiam. ## Predicted Entities `Normal`, `Boiling`, `Algae`, `NormalCement`, `NormalRain`, `BoilingNight`, `NormalNight` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_6_en_4.1.0_3.0_1660167065053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_6_en_4.1.0_3.0_1660167065053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_6", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_6", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_6| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Marathi DistilBertForMaskedLM Cased model (from DarshanDeshpande) author: John Snow Labs name: distilbert_embeddings_marathi date: 2022-12-12 tags: [mr, open_source, distilbert_embeddings, distilbertformaskedlm] task: Embeddings language: mr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `marathi-distilbert` is a Marathi model originally trained by `DarshanDeshpande`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_marathi_mr_4.2.4_3.0_1670865013879.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_marathi_mr_4.2.4_3.0_1670865013879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_marathi","mr") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(False) pipeline = Pipeline(stages=[documentAssembler, tokenizer, distilbert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_marathi","mr") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(false) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, distilbert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_marathi| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|mr| |Size:|247.8 MB| |Case sensitive:|false| ## References - https://huggingface.co/DarshanDeshpande/marathi-distilbert - https://github.com/DarshanDeshpande - https://www.linkedin.com/in/darshan-deshpande/ - https://github.com/Baras64 - http://www.linkedin.com/in/harsh-abhi --- layout: model title: Fast Neural Machine Translation Model from Tonga (Zambezi) to English author: John Snow Labs name: opus_mt_toi_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, toi, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `toi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_toi_en_xx_2.7.0_2.4_1609168217047.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_toi_en_xx_2.7.0_2.4_1609168217047.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_toi_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Authority Clause Binary Classifier author: John Snow Labs name: legclf_authority_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `authority` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `authority` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_authority_clause_en_1.0.0_3.2_1660122135152.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_authority_clause_en_1.0.0_3.2_1660122135152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_authority_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[authority]| |[other]| |[other]| |[authority]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_authority_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support authority 0.86 0.78 0.82 23 other 0.95 0.97 0.96 99 accuracy - - 0.93 122 macro-avg 0.90 0.88 0.89 122 weighted-avg 0.93 0.93 0.93 122 ``` --- layout: model title: Word2Vec Embeddings in Tosk Albanian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, als, open_source] task: Embeddings language: als edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_als_3.4.1_3.0_1647282325294.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_als_3.4.1_3.0_1647282325294.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+------------------+--------------------------------------------------------------+ |Drug |Treats |Pharmaceutical Action | +---------+------------------+--------------------------------------------------------------+ |Aklis |Hyperlipidemia |Hypertension:::Diabetic Kidney Disease:::Cerebrovascular... | |Dermovate|Lupus |Discoid Lupus Erythematosus:::Empeines:::Psoriasis:::Eczema...| |Diprivan |Infection |Laryngitis:::Pneumonia:::Pharyngitis | |Proventil|Addison's Disease |Allergic Conjunctivitis:::Anemia:::Ankylosing Spondylitis | +---------+------------------+--------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|drug_action_treatment_mapper| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|8.7 MB| --- layout: model title: Fast Neural Machine Translation Model from Yoruba to English author: John Snow Labs name: opus_mt_yo_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, yo, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `yo` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_yo_en_xx_2.7.0_2.4_1609167298178.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_yo_en_xx_2.7.0_2.4_1609167298178.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_yo_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Arabic BertForMaskedLM Cased model (from UBC-NLP) author: John Snow Labs name: bert_embeddings_marbertv2 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `MARBERTv2` is a Arabic model originally trained by `UBC-NLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_marbertv2_ar_4.2.4_3.0_1670015405976.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_marbertv2_ar_4.2.4_3.0_1670015405976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_marbertv2","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_marbertv2","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_marbertv2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|609.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/UBC-NLP/MARBERTv2 - https://aclanthology.org/2021.acl-long.551.pdf - https://github.com/UBC-NLP/marbert - https://doi.org/10.14288/SOCKEYE - https://www.tensorflow.org/tfrc --- layout: model title: Fast Neural Machine Translation Model from English to Kinyarwanda author: John Snow Labs name: opus_mt_en_rw date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, rw, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `rw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_rw_xx_2.7.0_2.4_1609169329842.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_rw_xx_2.7.0_2.4_1609169329842.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_rw| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Part of Speech for Indonesian author: John Snow Labs name: roberta_token_classifier_pos_tagger date: 2021-12-27 tags: [indonesian, roberta, pos, id, open_source] task: Part of Speech Tagging language: id edition: Spark NLP 3.3.4 spark_version: 2.4 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` and it's been fine-tuned on [indonlu's](https://hf.co/datasets/indonlu) POSP dataset for the Indonesian language, leveraging `RoBERTa` embeddings and `RobertaForTokenClassification` for POS tagging purposes. ## Predicted Entities `PPO`, `KUA`, `ADV`, `PRN`, `VBI`, `PAR`, `VBP`, `NNP`, `UNS`, `VBT`, `VBL`, `NNO`, `ADJ`, `PRR`, `PRK`, `CCN`, `$$$`, `ADK`, `ART`, `CSN`, `NUM`, `SYM`, `INT`, `NEG`, `PRI`, `VBE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_pos_tagger_id_3.3.4_2.4_1640589883082.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_pos_tagger_id_3.3.4_2.4_1640589883082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+---------+ |chunk |ner_label| +------+---------+ |Budi |NNO | |sedang|ADK | |pergi |VBI | |ke |PPO | |pasar |NNO | |. |SYM | +------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_pos_tagger| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|id| |Size:|466.2 MB| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/w11wo/indonesian-roberta-base-posp-tagger](https://huggingface.co/w11wo/indonesian-roberta-base-posp-tagger) ## Benchmarking ```bash label score f1 0.8893 Accuracy 0.9399 ``` --- layout: model title: Marathi ALBERT Embeddings (v1) author: John Snow Labs name: albert_embeddings_marathi_albert date: 2022-04-14 tags: [albert, embeddings, mr, open_source] task: Embeddings language: mr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `marathi-albert` is a Marathi model orginally trained by `l3cube-pune`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_marathi_albert_mr_3.4.2_3.0_1649954258953.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_marathi_albert_mr_3.4.2_3.0_1649954258953.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_marathi_albert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|mr| |Size:|45.1 MB| |Case sensitive:|false| ## References - https://huggingface.co/l3cube-pune/marathi-albert - https://github.com/l3cube-pune/MarathiNLP - https://arxiv.org/abs/2202.01159 --- layout: model title: Sentiment Analysis in Spanish author: John Snow Labs name: beto_sentiment date: 2022-10-11 tags: [beto, sentiment, bert, es, open_source] task: Text Classification language: es edition: Spark NLP 4.2.0 spark_version: [3.2, 3.0] supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Model trained with TASS 2020 corpus (around ~5k tweets) of several dialects of Spanish. Base model is BETO, a BERT model trained in Spanish. ## Predicted Entities `POS`, `NEG`, `NEU` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/beto_sentiment_es_4.2.0_3.2_1665504729919.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/beto_sentiment_es_4.2.0_3.2_1665504729919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------+------+ | text|result| +------------------+------+ |Te quiero. Te amo.| [POS]| +------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|beto_sentiment| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|412.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References https://github.com/finiteautomata/pysentimiento/ --- layout: model title: Detect PHI for Deidentification purposes (Spanish) author: John Snow Labs name: ner_deid_subentity date: 2022-01-18 tags: [deid, es, licensed] task: De-identification language: es edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotators allow for a generic model to be trained by using a Deep Learning architecture (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Spanish) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 13 entities. This NER model is trained with a combination of custom datasets, Spanish 2002 conLL, MeddoProf dataset and several data augmentation mechanisms. ## Predicted Entities `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `E-MAIL`, `USERNAME`, `LOCATION`, `ZIP`, `MEDICALRECORD`, `PROFESSION`, `PHONE`, `DOCTOR`, `AGE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEID_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/healthcare-nlp/04.1.Clinical_Multi_Language_Deidentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_es_3.3.4_3.0_1642512189785.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_es_3.3.4_3.0_1642512189785.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("word_embeddings") clinical_ner = medical.NerModel.pretrained("ner_deid_subentity", "es", "clinical/models")\ .setInputCols(["sentence","token","word_embeddings"])\ .setOutputCol("ner") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner]) text = [''' Antonio Pérez Juan, nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14/03/2020 y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. '''] data = spark.createDataFrame([text]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_deid_subentity", "es", "clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, clinical_ner)) val text = """Antonio Pérez Juan, nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14/03/2020 y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos.""" val data = Seq(text).toDS.toDF("text") val results = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.deid.subentity").predict(""" Antonio Pérez Juan, nacido en Cadiz, España. Aún no estaba vacunado, se infectó con Covid-19 el dia 14/03/2020 y tuvo que ir al Hospital. Fue tratado con anticuerpos monoclonales en la Clinica San Carlos. """) ```

## Results ```bash +------------+----------+ | token| ner_label| +------------+----------+ | Antonio| B-PATIENT| | Pérez| I-PATIENT| | Juan| I-PATIENT| | ,| O| | nacido| O| | en| O| | Cadiz|B-LOCATION| | ,| O| | España|B-LOCATION| | .| O| | Aún| O| | no| O| | estaba| O| | vacunado| O| | ,| O| | se| O| | infectó| O| | con| O| | Covid-19| O| | el| O| | dia| O| | 14| B-DATE| | de| I-DATE| | Marzo| I-DATE| | y| O| | tuvo| O| | que| O| | ir| O| | al| O| | Hospital| O| | Fue| O| | tratado| O| | con| O| | anticuerpos| O| |monoclonales| O| | en| O| | la| O| | Clinica|B-HOSPITAL| | San|I-HOSPITAL| | Carlos|I-HOSPITAL| | .| O| +------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, word_embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|15.0 MB| |Dependencies:|embeddings_sciwiki_300d| ## Data Source - Internal JSL annotated corpus - [Spanish conLL](https://www.clips.uantwerpen.be/conll2002/ner/data/) - [MeddoProf](https://temu.bsc.es/meddoprof/data/) ## Benchmarking ```bash label tp fp fn total precision recall f1 PATIENT 2088.0 201.0 178.0 2266.0 0.9122 0.9214 0.9168 HOSPITAL 302.0 43.0 85.0 387.0 0.8754 0.7804 0.8251 DATE 1837.0 33.0 20.0 1857.0 0.9824 0.9892 0.9858 ORGANIZATION 2498.0 477.0 649.0 3147.0 0.8397 0.7938 0.8161 MAIL 58.0 0.0 0.0 58.0 1.0 1.0 1.0 USERNAME 90.0 0.0 15.0 105.0 1.0 0.8571 0.9231 LOCATION 1866.0 391.0 354.0 2220.0 0.8268 0.8405 0.8336 ZIP 20.0 1.0 2.0 22.0 0.9524 0.9091 0.9302 MEDICALRECORD 111.0 5.0 20.0 131.0 0.9569 0.8473 0.8988 PROFESSION 270.0 96.0 134.0 404.0 0.7377 0.6683 0.7013 PHONE 108.0 11.0 8.0 116.0 0.9076 0.931 0.9191 DOCTOR 659.0 40.0 40.0 699.0 0.9428 0.9428 0.9428 AGE 302.0 53.0 61.0 363.0 0.8507 0.832 0.8412 macro - - - - - - 0.8872247 micro - - - - - - 0.8741892 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from vkmr) author: John Snow Labs name: distilbert_qa_vkmr_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `vkmr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkmr_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726538207.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_vkmr_base_uncased_finetuned_squad_en_4.0.0_3.0_1654726538207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkmr_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_vkmr_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_vkmr").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_vkmr_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/vkmr/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Personnel Management And Staff Remuneration Document Classifier (EURLEX) author: John Snow Labs name: legclf_personnel_management_and_staff_remuneration_bert date: 2023-03-06 tags: [en, legal, classification, clauses, personnel_management_and_staff_remuneration, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_personnel_management_and_staff_remuneration_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Personnel_Management_and_Staff_Remuneration or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Personnel_Management_and_Staff_Remuneration`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_personnel_management_and_staff_remuneration_bert_en_1.0.0_3.0_1678111663312.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_personnel_management_and_staff_remuneration_bert_en_1.0.0_3.0_1678111663312.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_personnel_management_and_staff_remuneration_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Personnel_Management_and_Staff_Remuneration]| |[Other]| |[Other]| |[Personnel_Management_and_Staff_Remuneration]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_personnel_management_and_staff_remuneration_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.93 0.87 0.90 47 Personnel_Management_and_Staff_Remuneration 0.89 0.94 0.92 53 accuracy - - 0.91 100 macro-avg 0.91 0.91 0.91 100 weighted-avg 0.91 0.91 0.91 100 ``` --- layout: model title: Detect Living Species(embeddings_scielo_300d) author: John Snow Labs name: ner_living_species_300 date: 2022-11-22 tags: [licensed, clinical, es, ner] task: Named Entity Recognition language: es edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Spanish which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `embeddings_scielo_300d` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_es_4.2.2_3.0_1669127690723.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_es_4.2.2_3.0_1669127690723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_300", "es","clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_300", "es","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new PipelineModel().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.living_species.300").predict("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_japanese_hiragana_vtuber", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_japanese_hiragana_vtuber", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_japanese_hiragana_vtuber| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Spanish RoBERTa Embeddings (Base, Using Sequence Length 512) author: John Snow Labs name: roberta_embeddings_bertin_base_gaussian_exp_512seqlen date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bertin-base-gaussian-exp-512seqlen` is a Spanish model orginally trained by `bertin-project`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_gaussian_exp_512seqlen_es_3.4.2_3.0_1649945655435.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_gaussian_exp_512seqlen_es_3.4.2_3.0_1649945655435.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bertin_base_gaussian_exp_512seqlen| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|234.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/bertin-project/bertin-base-gaussian-exp-512seqlen --- layout: model title: Fake News Classifier in Urdu author: John Snow Labs name: classifierdl_urduvec_fakenews date: 2021-12-29 tags: [urdu, fake_news, fake, ur, open_source] task: Text Classification language: ur edition: Spark NLP 3.3.1 spark_version: 3.0 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model determines if news articles in Urdu are real or fake. ## Predicted Entities `real`, `fake` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_UR_FAKENEWS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_UR_FAKENEWS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_urduvec_fakenews_ur_3.3.1_3.0_1640771335815.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_urduvec_fakenews_ur_3.3.1_3.0_1640771335815.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("news") \ .setOutputCol("document") tokenizer = Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") normalizer = Normalizer() \ .setInputCols(["token"]) \ .setOutputCol("normalized") lemma = LemmatizerModel.pretrained("lemma", "ur") \ .setInputCols(["normalized"]) \ .setOutputCol("lemma") embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") \ .setInputCols("document", "lemma") \ .setOutputCol("embeddings") embeddingsSentence = SentenceEmbeddings() \ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifierdl = ClassifierDLModel.pretrained("classifierdl_urduvec_fakenews", "ur") \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") urdu_fake_pipeline = Pipeline(stages=[document_assembler, tokenizer, normalizer, lemma, embeddings, embeddingsSentence, classifierdl]) light_pipeline = LightPipeline(urdu_fake_pipeline.fit(spark.createDataFrame([['']]).toDF("news"))) result = light_pipeline.annotate("ایک امریکی تھنک ٹینک نے خبردار کیا ہے کہ جیسے جیسے چین مصنوعی ذہانت (آرٹیفیشل انٹیلی جنس) کے میدان میں ترقی کر رہا ہے، دنیا کا اقتصادی اور عسکری توازن تبدیل ہو سکتا ہے۔") result["class"] ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val normalizer = Normalizer() .setInputCols(Array("token")) .setOutputCol("normalized") val lemma = LemmatizerModel.pretrained("lemma", "ur") .setInputCols(Array("normalized")) .setOutputCol("lemma") val embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") .setInputCols(Array("document", "lemma")) .setOutputCol("embeddings") val embeddingsSentence = SentenceEmbeddings() .setInputCols(Array("document", "embeddings")) .setOutputCol("sentence_embeddings") .setPoolingStrategy("AVERAGE") val classifier = ClassifierDLModel.pretrained("classifierdl_urduvec_fakenews", "ur") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val urdu_fake_pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, normalizer, lemma, embeddings, embeddingsSentence, classifier)) val light_pipeline = LightPipeline(urdu_fake_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) val result = light_pipeline.annotate("ایک امریکی تھنک ٹینک نے خبردار کیا ہے کہ جیسے جیسے چین مصنوعی ذہانت (آرٹیفیشل انٹیلی جنس) کے میدان میں ترقی کر رہا ہے، دنیا کا اقتصادی اور عسکری توازن تبدیل ہو سکتا ہے۔") ``` {:.nlu-block} ```python import nlu nlu.load("ur.classify.fakenews").predict("""ایک امریکی تھنک ٹینک نے خبردار کیا ہے کہ جیسے جیسے چین مصنوعی ذہانت (آرٹیفیشل انٹیلی جنس) کے میدان میں ترقی کر رہا ہے، دنیا کا اقتصادی اور عسکری توازن تبدیل ہو سکتا ہے۔""") ```

## Results ```bash ['real'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_urduvec_fakenews| |Compatibility:|Spark NLP 3.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|ur| |Size:|21.5 MB| ## Data Source Combination of multiple open source data sets. ## Benchmarking ```bash label precision recall f1-score support fake 0.77 0.70 0.73 415 real 0.71 0.77 0.74 387 accuracy 0.73 802 macro-avg 0.74 0.74 0.73 802 weighted-avg 0.74 0.73 0.73 802 ``` --- layout: model title: English BertForQuestionAnswering model (from tiennvcs) author: John Snow Labs name: bert_qa_bert_base_uncased_finetuned_infovqa date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-infovqa` is a English model orginally trained by `tiennvcs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_infovqa_en_4.0.0_3.0_1654180950608.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_infovqa_en_4.0.0_3.0_1654180950608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_finetuned_infovqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_finetuned_infovqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.infovqa.base_uncased.by_tiennvcs").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_finetuned_infovqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tiennvcs/bert-base-uncased-finetuned-infovqa --- layout: model title: Extract relations between effects of using multiple drugs (ReDL) author: John Snow Labs name: redl_drug_drug_interaction_biobert date: 2021-07-24 tags: [relation_extraction, en, licensed, clinical] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 2.4 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract potential improvements or harmful effects of Drug-Drug interactions (DDIs) when two or more drugs are taken at the same time or at a certain interval. ## Predicted Entities `DDI-advise`, `DDI-effect`, `DDI-false`, `DDI-int`, `DDI-mechanism` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_DRUG_DRUG_INT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_3.0.3_2.4_1627119817997.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_3.0.3_2.4_1627119817997.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ | relation|entity1|entity1_begin|entity1_end| chunk1|entity2|entity2_begin|entity2_end| chunk2|confidence| +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ |DDI-false| DRUG| 5| 17|carbamazepine| DRUG| 62| 73|aripiprazole|0.91685396| +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_drug_drug_interaction_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on DDI Extraction corpus. ## Benchmarking ```bash Relation Recall Precision F1 Support DDI-advise 0.758 0.874 0.812 211 DDI-effect 0.759 0.754 0.756 348 DDI-false 0.977 0.957 0.967 4097 DDI-int 0.175 0.458 0.253 63 DDI-mechanism 0.783 0.853 0.816 281 Avg. 0.690 0.779 0.721 - ``` --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli - EntityChunkEmbeddings) author: John Snow Labs name: sbiobertresolve_rxnorm_augmented_re date: 2022-02-09 tags: [rxnorm, licensed, en, clinical, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.4.0 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes without specifying the relations between the entities (relations are calculated on the fly inside the annotator) using sbiobert_base_cased_mli Sentence Bert Embeddings (EntityChunkEmbeddings). Embeddings used in this model are calculated with following weights : `{"DRUG": 0.8, "STRENGTH": 0.2, "ROUTE": 0.2, "FORM": 0.2}` . EntityChunkEmbeddings with those weights are required in the pipeline to get best result. ## Predicted Entities `RxNorm Codes`, `Concept Classes` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_re_en_3.4.0_2.4_1644395696788.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_augmented_re_en_3.4.0_2.4_1644395696788.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") sentence_detector = SentenceDetector() \ .setInputCols("documents") \ .setOutputCol("sentences") tokenizer = Tokenizer() \ .setInputCols("sentences") \ .setOutputCol("tokens") embeddings = WordEmbeddingsModel() \ .pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("embeddings") posology_ner_model = MedicalNerModel()\ .pretrained("ner_posology_large", "en", "clinical/models")\ .setInputCols(["sentences", "tokens", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentences", "tokens", "ner"])\ .setOutputCol("ner_chunks") pos_tager = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") dependency_parser = DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") drug_chunk_embeddings = EntityChunkEmbeddings()\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunks", "dependencies"])\ .setOutputCol("drug_chunk_embeddings")\ .setMaxSyntacticDistance(3)\ .setTargetEntities({"DRUG": ["STRENGTH", "ROUTE", "FORM"]})\ .setEntityWeights({"DRUG": 0.8, "STRENGTH": 0.2, "ROUTE": 0.2, "FORM": 0.2}) rxnorm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxnorm_augmented_re", "en", "clinical/models")\ .setInputCols(["drug_chunk_embeddings"])\ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") rxnorm_weighted_pipeline_re = Pipeline( stages = [ documenter, sentence_detector, tokenizer, embeddings, posology_ner_model, ner_converter, pos_tager, dependency_parser, drug_chunk_embeddings, rxnorm_resolver ]) sampleText = ["The patient was given metformin 500 mg, 2.5 mg of coumadin and then ibuprofen.", "The patient was given metformin 400 mg, coumadin 5 mg, coumadin, amlodipine 10 MG"] data_df = spark.createDataFrame(sample_df) results = rxnorm_weighted_pipeline_re.fit(data_df).transform(data_df) ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val sentence_detector = SentenceDetector() .setInputCols("documents") .setOutputCol("sentences") val tokenizer = Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val embeddings = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val posology_ner_model = MedicalNerModel() .pretrained("ner_posology_large", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverterInternal() .setInputCols(Array("sentences", "tokens", "ner")) .setOutputCol("ner_chunks") val pos_tager = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") val drug_chunk_embeddings = EntityChunkEmbeddings() .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunks", "dependencies")) .setOutputCol("drug_chunk_embeddings") .setMaxSyntacticDistance(3) .setTargetEntities({"DRUG": ["STRENGTH", "ROUTE", "FORM"]}) .setEntityWeights({"DRUG": 0.8, "STRENGTH": 0.2, "ROUTE": 0.2, "FORM": 0.2} val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm_augmented_re", "en", "clinical/models") .setInputCols(Array("drug_chunk_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val rxnorm_weighted_pipeline_re = new PipelineModel().setStages(Array(documenter, sentence_detector, tokenizer, embeddings, posology_ner_model, ner_converter, pos_tager, dependency_parser, drug_chunk_embeddings, rxnorm_re)) val light_model = LightPipeline(rxnorm_weighted_pipeline_re) vat sampleText = Array("The patient was given metformin 500 mg, 2.5 mg of coumadin and then ibuprofen.", "The patient was given metformin 400 mg, coumadin 5 mg, coumadin, amlodipine 10 MG") val results = rxnorm_weighted_pipeline_re.fit(sampleText).transform(sampleText) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm.augmented_re").predict("""The patient was given metformin 400 mg, coumadin 5 mg, coumadin, amlodipine 10 MG""") ```

## Results ```bash +-----+----------------+--------------------------+--------------------------------------------------+ |index| chunk|rxnorm_code_weighted_08_re| Concept_Name| +-----+----------------+--------------------------+--------------------------------------------------+ | 0|metformin 500 mg| 860974|metformin hydrochloride 500 MG:::metformin 500 ...| | 0| 2.5 mg coumadin| 855313|warfarin sodium 2.5 MG [Coumadin]:::warfarin so...| | 0| ibuprofen| 1747293|ibuprofen Injection:::ibuprofen Pill:::ibuprofe...| | 1|metformin 400 mg| 332809|metformin 400 MG:::metformin 250 MG Oral Tablet...| | 1| coumadin 5 mg| 855333|warfarin sodium 5 MG [Coumadin]:::warfarin sodi...| | 1| coumadin| 202421|Coumadin:::warfarin sodium 2 MG/ML Injectable S...| | 1|amlodipine 10 MG| 308135|amlodipine 10 MG Oral Tablet:::amlodipine 10 MG...| +-----+----------------+--------------------------+--------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_augmented_re| |Compatibility:|Healthcare NLP 3.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Size:|759.7 MB| |Case sensitive:|false| --- layout: model title: English image_classifier_vit_grain ViTForImageClassification from roydcarlson author: John Snow Labs name: image_classifier_vit_grain date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_grain` is a English model originally trained by roydcarlson. ## Predicted Entities `teff`, `buckwheat`, `barley`, `wheat`, `millet` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_grain_en_4.1.0_3.0_1660169441159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_grain_en_4.1.0_3.0_1660169441159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_grain", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_grain", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_grain| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Stopwords Remover for Korean language (63 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ko, open_source] task: Stop Words Removal language: ko edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ko_3.4.1_3.0_1646673221687.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ko_3.4.1_3.0_1646673221687.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------+ |result | +--------------------------------+ |[당신은, 나보다, 낫지, 않습니다]| +--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ko| |Size:|1.6 KB| --- layout: model title: Word Embeddings for Hebrew (hebrew_cc_300d) author: John Snow Labs name: hebrew_cc_300d date: 2020-12-09 task: Embeddings language: he edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [embeddings, open_source, he] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/hebrew_cc_300d_he_2.7.0_2.4_1607518856144.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/hebrew_cc_300d_he_2.7.0_2.4_1607518856144.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['כמו גם התקפות והאשמות נגד בראון']], ["text"])) ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("hebrew_cc_300d", "he") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("כמו גם התקפות והאשמות נגד בראון").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["כמו גם התקפות והאשמות נגד בראון"] hebrevec_df = nlu.load('he.embed.cbow_300d').predict(text, output_level="token") hebrevec_df ```

{:.h2_title} ## Results The model gives 300 dimensional Word2Vec feature vector outputs per token. ```bash | he_embed_cbow_300d_embeddings | token |----------------------------------------------------|------- | [0.01140000019222498, 0.005900000222027302, 0.... כמו | [-0.06199999898672104, 0.04879999905824661, 0.... גם | [0.041600000113248825, 0.045099999755620956, 0... התקפות | [0.01489999983459711, 0.024800000712275505, -0... והאשמות | [-0.049800001084804535, 0.05260000005364418, -... נגד | [0.01209999993443489, -0.012600000016391277, -... בראון ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|hebrew_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|he| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from [https://fasttext.cc/docs/en/crawl-vectors.html](https://fasttext.cc/docs/en/crawl-vectors.html) --- layout: model title: Legal Participations Clause Binary Classifier author: John Snow Labs name: legclf_participations_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `participations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `participations` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_participations_clause_en_1.0.0_3.2_1660122818723.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_participations_clause_en_1.0.0_3.2_1660122818723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_participations_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[participations]| |[other]| |[other]| |[participations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_participations_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.92 0.98 0.95 57 participations 0.93 0.74 0.82 19 accuracy - - 0.92 76 macro-avg 0.93 0.86 0.89 76 weighted-avg 0.92 0.92 0.92 76 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from kizunasunhy) author: John Snow Labs name: distilbert_qa_kizunasunhy_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `kizunasunhy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_kizunasunhy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771843588.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_kizunasunhy_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771843588.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kizunasunhy_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kizunasunhy_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_kizunasunhy_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kizunasunhy/distilbert-base-uncased-finetuned-squad --- layout: model title: Translate English to Gilbertese Pipeline author: John Snow Labs name: translate_en_gil date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, gil, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `gil` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_gil_xx_2.7.0_2.4_1609686096384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_gil_xx_2.7.0_2.4_1609686096384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_gil", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_gil", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.gil').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_gil| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Polish to English author: John Snow Labs name: opus_mt_pl_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pl, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `pl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_pl_en_xx_2.7.0_2.4_1609167932979.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_pl_en_xx_2.7.0_2.4_1609167932979.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_pl_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Normalized Genes and Human Phenotypes (biobert) author: John Snow Labs name: ner_human_phenotype_go_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect normalized mentions of genes (go) and human phenotypes (hp) in medical text. ## Predicted Entities `HP`, `GO` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HUMAN_PHENOTYPE_GO_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_biobert_en_3.0.0_3.0_1617260627136.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_go_biobert_en_3.0.0_3.0_1617260627136.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_human_phenotype_go_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_human_phenotype_go_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phenotype.go_biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_go_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash entity tp fp fn total precision recall f1 GO 7637.0 579.0 441.0 8078.0 0.9295 0.9454 0.9374 HP 1463.0 273.0 222.0 1685.0 0.8427 0.8682 0.8553 macro - - - - - - 0.89635 micro - - - - - - 0.92323 ``` --- layout: model title: Arabic BertForMaskedLM Large Cased model (from aubmindlab) author: John Snow Labs name: bert_embeddings_large_arabertv2 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-arabertv2` is a Arabic model originally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabertv2_ar_4.2.4_3.0_1670019805854.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_large_arabertv2_ar_4.2.4_3.0_1670019805854.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabertv2","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_large_arabertv2","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_large_arabertv2| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|1.4 GB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-large-arabertv2 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-large-arabertv2/blob/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: English RobertaForQuestionAnswering Cased model (from eAsyle) author: John Snow Labs name: roberta_qa_testabsa date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `testABSA` is a English model originally trained by `eAsyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_testabsa_en_4.3.0_3.0_1674224267018.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_testabsa_en_4.3.0_3.0_1674224267018.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_testabsa","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_testabsa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_testabsa| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|426.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/eAsyle/testABSA --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el16_en_4.3.0_3.0_1675119492656.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el16_en_4.3.0_3.0_1675119492656.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|208.2 MB| ## References - https://huggingface.co/google/t5-efficient-small-el16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Language Detection & Identification Pipeline - 95 Languages author: John Snow Labs name: detect_language_95 date: 2020-12-05 task: [Pipeline Public, Language Detection, Sentence Detection] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, pipeline, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNN architectures in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This pipeline can detect the following languages: ## Predicted Entities `Afrikaans`, `Amharic`, `Aragonese`, `Arabic`, `Assamese`, `Azerbaijani`, `Belarusian`, `Bulgarian`, `Bengali`, `Breton`, `Bosnian`, `Catalan`, `Czech`, `Welsh`, `Danish`, `German`, `Greek`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Basque`, `Persian`, `Finnish`, `Faroese`, `French`, `Irish`, `Galician`, `Gujarati`, `Hebrew`, `Hindi`, `Croatian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Icelandic`, `Italian`, `Japanese`, `Javanese`, `Georgian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Kurdish`, `Kyrgyz`, `Latin`, `Luxembourgish`, `Lao`, `Lithuanian`, `Latvian`, `Malagasy`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Malay`, `Maltese`, `Nepali`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Occitan`, `Odia (Oriya)`, `Punjabi (Eastern)`, `Polish`, `Pashto`, `Portuguese`, `Quechua`, `Romanian`, `Russian`, `Northern Sami`, `Sinhala`, `Slovak`, `Slovenian`, `Albanian`, `Serbian`, `Swedish`, `Swahili`, `Tamil`, `Telugu`, `Thai`, `Tagalog`, `Turkish`, `Tatar`, `Uyghur`, `Ukrainian`, `Urdu`, `Vietnamese`, `Volapük`, `Walloon`, `Xhosa`, `Chinese`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/LANGUAGE_DETECTOR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/detect_language_95_xx_2.7.0_2.4_1607185479059.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/detect_language_95_xx_2.7.0_2.4_1607185479059.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("detect_language_95", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("detect_language_95", lang = "xx) pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` {:.nlu-block} ```python import nlu text = ["French author who helped pioneer the science-fiction genre."] lang_df = nlu.load("xx.classify.lang.95").predict(text) lang_df ```

## Results ```bash {'document': ['French author who helped pioneer the science-fiction genre.'], 'language': ['en']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|detect_language_95| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - LanguageDetectorDL --- layout: model title: English BertForQuestionAnswering model (from vuiseng9) author: John Snow Labs name: bert_qa_bert_l_squadv1.1_sl384 date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-l-squadv1.1-sl384` is a English model orginally trained by `vuiseng9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_l_squadv1.1_sl384_en_4.0.0_3.0_1654536116442.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_l_squadv1.1_sl384_en_4.0.0_3.0_1654536116442.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_l_squadv1.1_sl384","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_l_squadv1.1_sl384","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.sl384.by_vuiseng9").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_l_squadv1.1_sl384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vuiseng9/bert-l-squadv1.1-sl384 --- layout: model title: Pipeline to Detect Problem, Test and Treatment author: John Snow Labs name: ner_healthcare_pipeline date: 2022-03-22 tags: [licensed, ner, healthcare, treatment, problem, test, en, clinical] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_healthcare](https://nlp.johnsnowlabs.com/2021/04/21/ner_healthcare_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_pipeline_en_3.4.1_3.0_1647943495587.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_pipeline_en_3.4.1_3.0_1647943495587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.healthcare_pipeline").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG .""") ```

## Results ```bash +-----------------------------+---------+ |chunks |entities | +-----------------------------+---------+ |gestational diabetes mellitus|PROBLEM | |type two diabetes mellitus |PROBLEM | |HTG-induced pancreatitis |PROBLEM | |an acute hepatitis |PROBLEM | |obesity |PROBLEM | |a body mass index |TEST | |BMI |TEST | |polyuria |PROBLEM | |polydipsia |PROBLEM | |poor appetite |PROBLEM | |vomiting |PROBLEM | |amoxicillin |TREATMENT| |a respiratory tract infection|PROBLEM | |metformin |TREATMENT| |glipizide |TREATMENT| +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.5 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Japanese Bert Embeddings (Base, Character Tokenization, v2) author: John Snow Labs name: bert_embeddings_bert_base_japanese_char_v2 date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-char-v2` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_v2_ja_3.4.2_3.0_1649674322814.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_char_v2_ja_3.4.2_3.0_1649674322814.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_char_v2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|340.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-char-v2 - https://github.com/google-research/bert - https://pypi.org/project/unidic-lite/ - https://github.com/cl-tohoku/bert-japanese/tree/v2.0 - https://taku910.github.io/mecab/ - https://github.com/neologd/mecab-ipadic-neologd - https://github.com/polm/fugashi - https://github.com/polm/unidic-lite - https://www.tensorflow.org/tfrc/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: Stopwords Remover for Malayalam language (9 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ml, open_source] task: Stop Words Removal language: ml edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ml_3.4.1_3.0_1646672962533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ml_3.4.1_3.0_1646672962533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------+ |result | +---------------------------------+ |[നിങ്ങൾ, എന്നെക്കാൾ, മികച്ചവനല്ല]| +---------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ml| |Size:|1.4 KB| --- layout: model title: Moldavian, Moldovan, Romanian asr_romanian_wav2vec2 TFWav2Vec2ForCTC from gigant author: John Snow Labs name: asr_romanian_wav2vec2 date: 2022-09-25 tags: [wav2vec2, ro, audio, open_source, asr] task: Automatic Speech Recognition language: ro edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_romanian_wav2vec2` is a Moldavian, Moldovan, Romanian model originally trained by gigant. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_romanian_wav2vec2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_romanian_wav2vec2_ro_4.2.0_3.0_1664098158164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_romanian_wav2vec2_ro_4.2.0_3.0_1664098158164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_romanian_wav2vec2", "ro")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_romanian_wav2vec2", "ro") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_romanian_wav2vec2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ro| |Size:|1.2 GB| --- layout: model title: Legal Powers Clause Binary Classifier author: John Snow Labs name: legclf_powers_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `powers` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `powers` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_powers_clause_en_1.0.0_3.2_1660122856675.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_powers_clause_en_1.0.0_3.2_1660122856675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_powers_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[powers]| |[other]| |[other]| |[powers]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_powers_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.99 0.98 307 powers 0.97 0.93 0.95 124 accuracy - - 0.97 431 macro-avg 0.97 0.96 0.96 431 weighted-avg 0.97 0.97 0.97 431 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_dl16 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-dl16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl16_en_4.3.0_3.0_1675118638338.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_dl16_en_4.3.0_3.0_1675118638338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_dl16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_dl16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_dl16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|228.3 MB| ## References - https://huggingface.co/google/t5-efficient-small-dl16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl48 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl48` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl48_en_4.3.0_3.0_1675123074393.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl48_en_4.3.0_3.0_1675123074393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl48","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl48","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl48| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|742.0 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl48 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Resolve Company Names to Tickers using Wikidata author: John Snow Labs name: finel_wiki_parentorgs_ticker date: 2023-01-18 tags: [en, licensed] task: Entity Resolution language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model helps you retrieve the TICKER of a company using a previously detected ORG entity with NER. It also retrieves the normalized company name as per Wikidata, which can be retrieved from `aux_label` column in metadata. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finel_wiki_parentorgs_ticker_en_1.0.0_3.0_1674038769879.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finel_wiki_parentorgs_ticker_en_1.0.0_3.0_1674038769879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk") \ .setOutputCol("sentence_embeddings") resolver = finance.SentenceEntityResolverModel.pretrained("finel_wiki_parentorgs_tickers", "en", "finance/models")\ .setInputCols(["sentence_embeddings"]) \ .setOutputCol("normalized_name")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = nlp.Pipeline( stages = [ documentAssembler, embeddings, resolver ]) lp = nlp.LightPipeline(pipelineModel) test_pred = lp.fullAnnotate('Alphabet Incorporated') print(test_pred[0]['normalized_name'][0].result) print(test_pred[0]['normalized_name'][0].metadata['all_k_aux_labels'].split(':::')[0]) ```

## Results ```bash GOOGL Aux data: Alphabet Inc. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finel_wiki_parentorgs_ticker| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[original_company_name]| |Language:|en| |Size:|2.8 MB| |Case sensitive:|false| ## References Wikipedia dump about company subsidiaries --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav TFWav2Vec2ForCTC from vai6hav author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav` is a English model originally trained by vai6hav. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_en_4.2.0_3.0_1664112729220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav_en_4.2.0_3.0_1664112729220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_vai6hav| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: BERT Sequence Classification - Identify Trec Data Classes author: John Snow Labs name: bert_sequence_classifier_trec_coarse date: 2021-11-06 tags: [bert_for_sequence_classification, trec, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is imported from `Hugging Face-models` and it is a simple base BERT model trained on the "trec" dataset. ## Predicted Entities `DESC`, `ENTY`, `HUM`, `NUM`, `ABBR`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_trec_coarse_en_3.3.2_2.4_1636229841055.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_trec_coarse_en_3.3.2_2.4_1636229841055.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['LOC'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_trec_coarse| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[label]| |Language:|en| |Case sensitive:|true| ## Data Source [https://huggingface.co/aychang/bert-base-cased-trec-coarse](https://huggingface.co/aychang/bert-base-cased-trec-coarse) ## Benchmarking ```bash epoch: 2.0, eval_loss: 0.138086199760437 eval_runtime: 1.6132, eval_samples_per_second: 309.94 +------------+-------+-----------------+--------------+ | entity|eval_f1| eval_precision| eval_recall| +------------+-------+-----------------+--------------+ | DESC| 0.981| 0.985| 0.978| | ENTY| 0.944| 0.988| 0.904| | ABBR| 1.| 1.| 1.| | HUM| 0.992| 0.984| 1.| | NUM| 0.969| 0.941| 1.| | LOC| 0.981| 0.975| 0.987| +------------+-------+-----------------+--------------+ eval_accuracy: 0.974 ``` --- layout: model title: Detect Adverse Drug Events (MedicalBertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_ade_binary date: 2022-07-27 tags: [clinical, ade, licensed, public_health, token_classification, ner, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse reactions of drugs in texts excahnged over twitter. This model is trained with the `BertForTokenClassification` method from the transformers library and imported into Spark NLP. ## Predicted Entities `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_binary_en_4.0.0_3.0_1658906410540.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_ade_binary_en_4.0.0_3.0_1658906410540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_ade_binary", "en", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("label")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["sentence","token","label"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame(["I used to be on paxil but that made me more depressed and prozac made me angry", "Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs."], StringType()).toDF("text") result = model.transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_ade_binary", "en", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("label") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "label")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter)) val data = Seq(Array("I used to be on paxil but that made me more depressed and prozac made me angry", "Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs.")).toDS().toDF("text") val result = model.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_token.ner_ade_bert").predict("""Maybe cos of the insulin blocking effect of seroquel but i do feel sugar crashes when eat fast carbs.""") ```

## Results ```bash +-------------+---------+ |chunk |ner_label| +-------------+---------+ |depressed |ADE | |angry |ADE | |sugar crashes|ADE | +-------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_ade_binary| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## Benchmarking ```bash label precision recall f1-score support B-ADE 0.89 0.88 0.88 3720 I-ADE 0.85 0.84 0.84 3145 O 0.98 0.98 0.98 26963 accuracy - - 0.95 33828 macro-avg 0.90 0.90 0.90 33828 weighted-avg 0.95 0.95 0.95 33828 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Vietnamese author: John Snow Labs name: opus_mt_en_vi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, vi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `vi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_vi_xx_2.7.0_2.4_1609170438037.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_vi_xx_2.7.0_2.4_1609170438037.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_vi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal NER (Signers) author: John Snow Labs name: legner_signers date: 2022-08-16 tags: [en, legal, ner, agreements, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Legal NER Model, aimed to process the last page of the agreements when information can be found about: - People Signing the document; - Title of those people in their companies; - Company (Party) they represent; ## Predicted Entities `SIGNING_TITLE`, `SIGNING_PERSON`, `PARTY` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/LEGALNER_SIGNERS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_signers_en_1.0.0_3.2_1660646474494.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_signers_en_1.0.0_3.2_1660646474494.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+----------------+ | token| ner_label| +-----------+----------------+ | VENDOR| O| | :| O| |VENDINGDATA| B-PARTY| |CORPORATION| O| | ,| O| | a| O| | Nevada| O| |corporation| O| | By| O| | :| O| | /s/| O| | Steven|B-SIGNING_PERSON| | J|I-SIGNING_PERSON| | .|I-SIGNING_PERSON| | Blad|I-SIGNING_PERSON| | Its| O| | :| O| | Steven|B-SIGNING_PERSON| | J|I-SIGNING_PERSON| | .|I-SIGNING_PERSON| | Blad|I-SIGNING_PERSON| | CEO| B-SIGNING_TITLE| |DISTRIBUTOR| O| | :| O| | TECHNICAL| B-PARTY| | CASINO| I-PARTY| | SUPPLIES| I-PARTY| | LTD| I-PARTY| | ,| O| | an| O| | English| O| | company| O| | By| O| | :| O| | /s/| O| | David|B-SIGNING_PERSON| | K|I-SIGNING_PERSON| | .|I-SIGNING_PERSON| | Heap|I-SIGNING_PERSON| | Its| O| | :| O| | David|B-SIGNING_PERSON| | K|I-SIGNING_PERSON| | .|I-SIGNING_PERSON| | Heap|I-SIGNING_PERSON| | Chief| B-SIGNING_TITLE| | Executive| I-SIGNING_TITLE| | Officer| I-SIGNING_TITLE| | -| O| | 15| O| | -| O| +-----------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_signers| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.5 MB| ## References Manual annotations on CUAD dataset ## Benchmarking ```bash label tp fp fn prec rec f1 I-PARTY 366 26 39 0.93367344 0.9037037 0.91844416 I-SIGNING_TITLE 41 0 4 1.0 0.9111111 0.95348835 I-SIGNING_PERSON 115 10 13 0.92 0.8984375 0.9090909 B-SIGNING_PERSON 46 3 11 0.93877554 0.80701756 0.8679246 B-PARTY 122 14 28 0.89705884 0.81333333 0.85314685 B-SIGNING_TITLE 26 0 2 1.0 0.9285714 0.9629629 Macro-average 716 53 97 0.9482513 0.8770291 0.91125065 Micro-average 716 53 97 0.9310793 0.8806888 0.9051833 ``` --- layout: model title: English BertForTokenClassification Cased model (from Wanjiru) author: John Snow Labs name: bert_token_classifier_autotrain_gro_ner date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain_gro_ner` is a English model originally trained by `Wanjiru`. ## Predicted Entities `METRIC`, `ITEM` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_gro_ner_en_4.2.4_3.0_1669814750536.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_autotrain_gro_ner_en_4.2.4_3.0_1669814750536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_gro_ner","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_autotrain_gro_ner","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_autotrain_gro_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Wanjiru/autotrain_gro_ner --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_large_data_seed_0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-data-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_data_seed_0_en_4.0.0_3.0_1655736575851.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_data_seed_0_en_4.0.0_3.0_1655736575851.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_data_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_data_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.large_seed_0.by_anas-awadalla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_data_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-large-data-seed-0 --- layout: model title: English asr_wav2vec2_large_xlsr_coraa_portuguese_cv8 TFWav2Vec2ForCTC from lgris author: John Snow Labs name: asr_wav2vec2_large_xlsr_coraa_portuguese_cv8 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_coraa_portuguese_cv8` is a English model originally trained by lgris. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_en_4.2.0_3.0_1664043287384.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_coraa_portuguese_cv8_en_4.2.0_3.0_1664043287384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_coraa_portuguese_cv8", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_coraa_portuguese_cv8", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_coraa_portuguese_cv8| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from mrm8488) author: John Snow Labs name: roberta_qa_finetuned_squadv1 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_squadv1_en_4.3.0_3.0_1674210647091.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_squadv1_en_4.3.0_3.0_1674210647091.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_squadv1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_squadv1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|307.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/distilroberta-finetuned-squadv1 --- layout: model title: Oncology Pipeline for Biomarkers author: John Snow Labs name: oncology_biomarker_pipeline date: 2022-12-01 tags: [licensed, pipeline, oncology, biomarker, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition, Assertion Status and Relation Extraction models to extract information from oncology texts. This pipeline focuses on entities related to biomarkers. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.2.2_3.0_1669902355525.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_biomarker_pipeline_en_4.2.2_3.0_1669902355525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") pipeline.annotate("Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_biomarker_pipeline", "en", "clinical/models") val result = pipeline.fullAnnotate("""Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.""")(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_biomarker.pipeline").predict("""Immunohistochemistry was negative for thyroid transcription factor-1 and napsin A. The test was positive for ER and PR, and negative for HER2.""") ```

## Results ```bash ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_oncology_biomarker_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Biomarker | ******************** ner_oncology_test_wip results ******************** | chunk | ner_label | |:-------------------------------|:-----------------| | Immunohistochemistry | Pathology_Test | | negative | Biomarker_Result | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Result | | HER2 | Oncogene | ******************** ner_biomarker results ******************** | chunk | ner_label | |:-------------------------------|:----------------------| | Immunohistochemistry | Test | | negative | Biomarker_Measurement | | thyroid transcription factor-1 | Biomarker | | napsin A | Biomarker | | positive | Biomarker_Measurement | | ER | Biomarker | | PR | Biomarker | | negative | Biomarker_Measurement | | HER2 | Biomarker | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:------------| | Immunohistochemistry | Pathology_Test | Past | | thyroid transcription factor-1 | Biomarker | Present | | napsin A | Biomarker | Present | | ER | Biomarker | Present | | PR | Biomarker | Present | | HER2 | Oncogene | Present | ******************** assertion_oncology_test_binary_wip results ******************** | chunk | ner_label | assertion | |:-------------------------------|:---------------|:----------------| | Immunohistochemistry | Pathology_Test | Medical_History | | thyroid transcription factor-1 | Biomarker | Medical_History | | napsin A | Biomarker | Medical_History | | ER | Biomarker | Medical_History | | PR | Biomarker | Medical_History | | HER2 | Oncogene | Medical_History | ******************** re_oncology_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_related_to | | negative | Biomarker_Result | napsin A | Biomarker | is_related_to | | positive | Biomarker_Result | ER | Biomarker | is_related_to | | positive | Biomarker_Result | PR | Biomarker | is_related_to | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_related_to | ******************** re_oncology_granular_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | O | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | is_finding_of | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ******************** re_oncology_biomarker_result_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------------------|:-----------------|:-------------------------------|:-----------------|:--------------| | Immunohistochemistry | Pathology_Test | negative | Biomarker_Result | is_finding_of | | negative | Biomarker_Result | thyroid transcription factor-1 | Biomarker | is_finding_of | | negative | Biomarker_Result | napsin A | Biomarker | is_finding_of | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | | positive | Biomarker_Result | HER2 | Oncogene | O | | ER | Biomarker | negative | Biomarker_Result | O | | PR | Biomarker | negative | Biomarker_Result | O | | negative | Biomarker_Result | HER2 | Oncogene | is_finding_of | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_biomarker_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - RelationExtractionModel - RelationExtractionModel --- layout: model title: Legal Termination Clause Binary Classifier (CUAD dataset, USE version) author: John Snow Labs name: legclf_cuad_termination_clause date: 2022-11-09 tags: [termination, en, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `termination` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. This version was trained with Universal Sentence Encoder. There is another version using Sentence Bert, called `legclf_sbert_cuad_termination_clause` If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. There are other models in this dataset with similar title, but the difference is the dataset it was trained on. This one was trained with `cuad` dataset. ## Predicted Entities `termination`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_termination_clause_en_1.0.0_3.0_1667994003805.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_termination_clause_en_1.0.0_3.0_1667994003805.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_cuad_termination_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([[" ---------------------\n\n This Agreement may be terminated immediately by Developer..."]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[termination]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_termination_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References In-house annotations on CUAD dataset ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.97 0.99 35 termination 0.98 1.00 0.99 44 accuracy - - 0.99 79 macro-avg 0.99 0.99 0.99 79 weighted-avg 0.99 0.99 0.99 79 ``` --- layout: model title: Legal Miscellaneous provisions Clause Binary Classifier (md) author: John Snow Labs name: legclf_miscellaneous_provisions_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `miscellaneous-provisions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `miscellaneous-provisions` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_miscellaneous_provisions_md_en_1.0.0_3.0_1673460283239.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_miscellaneous_provisions_md_en_1.0.0_3.0_1673460283239.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_miscellaneous_provisions_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[miscellaneous-provisions]| |[other]| |[other]| |[miscellaneous-provisions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_miscellaneous_provisions_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support fees-and-expenses 0.92 0.89 0.91 27 other 0.93 0.95 0.94 39 accuracy 0.92 66 macro avg 0.92 0.92 0.92 66 weighted avg 0.92 0.92 0.92 66 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from georgio) author: John Snow Labs name: distilbert_qa_georgio_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `georgio`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_georgio_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770783179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_georgio_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770783179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_georgio_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_georgio_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_georgio_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/georgio/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Coal And Mining Industries Document Classifier (EURLEX) author: John Snow Labs name: legclf_coal_and_mining_industries_bert date: 2023-03-06 tags: [en, legal, classification, clauses, coal_and_mining_industries, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_coal_and_mining_industries_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Coal_and_Mining_Industries or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Coal_and_Mining_Industries`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_coal_and_mining_industries_bert_en_1.0.0_3.0_1678111573002.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_coal_and_mining_industries_bert_en_1.0.0_3.0_1678111573002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_coal_and_mining_industries_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Coal_and_Mining_Industries]| |[Other]| |[Other]| |[Coal_and_Mining_Industries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_coal_and_mining_industries_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Coal_and_Mining_Industries 0.91 0.88 0.90 34 Other 0.83 0.86 0.84 22 accuracy - - 0.88 56 macro-avg 0.87 0.87 0.87 56 weighted-avg 0.88 0.88 0.88 56 ``` --- layout: model title: Mapping Company Names to Edgar Database author: John Snow Labs name: legmapper_edgar_companyname date: 2022-08-18 tags: [en, legal, companies, edgar, data, augmentation, licensed] task: Chunk Mapping language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Chunk Mapper model allows you to, given a detected Organization with any NER model, augment it with information available in the SEC Edgar database. Some of the fields included in this Chunk Mapper are: - IRS number - Sector - Former names - Address, Phone, State - Dates where the company submitted filings - etc. IMPORTANT: Chunk Mappers work with exact matches, so before using Chunk Mapping, you need to carry out Company Name Normalization to get how the company name is stored in Edgar. To do this, use Entity Linking, more especifically the `finel_edgar_companynames` model, with the Organization Name extracted by any NER model. You will get the normalized version (by Edgar standards) of the name, which you can send to this model for data augmentation. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FIN_LEG_COMPANY_AUGMENTATION/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legmapper_edgar_companyname_en_1.0.0_3.2_1660817357103.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legmapper_edgar_companyname_en_1.0.0_3.2_1660817357103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = legal.NerModel.pretrained("legner_orgs_prods_alias", "en", "legal/models")\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") # Optional: To normalize the ORG name using NASDAQ data before the mapping ########################################################################## chunkToDoc = nlp.Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") chunk_embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("ner_chunk_doc") \ .setOutputCol("sentence_embeddings") use_er_model = legal.SentenceEntityResolverModel.pretrained("legel_edgar_company_name", "en", "legal/models") \ .setInputCols(["ner_chunk_doc", "sentence_embeddings"]) \ .setOutputCol("normalized")\ .setDistanceFunction("EUCLIDEAN") ########################################################################## cm = legal.ChunkMapperModel()\ .pretrained("legmapper_edgar_companyname", "en", "legal/models")\ .setInputCols(["normalized"])\ .setOutputCol("mappings") # or ner_chunk for non normalized versions nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner_model, ner_converter, chunkToDoc, chunk_embeddings, use_er_model, cm ]) text = """NIKE Inc is an American multinational corporation that is engaged in the design, development, manufacturing, and worldwide marketing and sales of footwear, apparel, equipment, accessories, and services""" test_data = spark.createDataFrame([[text]]).toDF("text") model = nlpPipeline.fit(test_data) lp = nlp.LightPipeline(model) lp.annotate(text) ```

## Results ```bash {"mappings": [["labeled_dependency", 0, 22, "Jamestown Invest 1, LLC", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "name", "all_relations": ""}], ["labeled_dependency", 0, 22, "REAL ESTATE INVESTMENT TRUSTS [6798]", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "sic", "all_relations": ""}], ["labeled_dependency", 0, 22, "6798", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "sic_code", "all_relations": ""}], ["labeled_dependency", 0, 22, "831529368", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "irs_number", "all_relations": ""}], ["labeled_dependency", 0, 22, "1231", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "fiscal_year_end", "all_relations": ""}], ["labeled_dependency", 0, 22, "GA", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "state_location", "all_relations": ""}], ["labeled_dependency", 0, 22, "DE", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "state_incorporation", "all_relations": ""}], ["labeled_dependency", 0, 22, "PONCE CITY MARKET", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "business_street", "all_relations": ""}], ["labeled_dependency", 0, 22, "ATLANTA", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "business_city", "all_relations": ""}], ["labeled_dependency", 0, 22, "GA", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "business_state", "all_relations": ""}], ["labeled_dependency", 0, 22, "30308", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "business_zip", "all_relations": ""}], ["labeled_dependency", 0, 22, "7708051000", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "business_phone", "all_relations": ""}], ["labeled_dependency", 0, 22, "Jamestown Atlanta Invest 1, LLC", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "former_name", "all_relations": ""}], ["labeled_dependency", 0, 22, "20180824", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "former_name_date", "all_relations": ""}], ["labeled_dependency", 0, 22, "2019-11-21", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "date", "all_relations": "2019-10-24:::2019-11-25:::2019-11-12:::2022-01-13:::2022-03-31:::2022-04-11:::2022-07-12:::2022-06-30:::2021-01-14:::2021-04-06:::2021-03-31:::2021-04-28:::2021-06-30:::2021-09-10:::2021-09-22:::2021-09-30:::2021-10-08:::2020-03-16:::2021-12-30:::2020-04-06:::2020-04-29:::2020-06-12:::2020-07-20:::2020-07-07:::2020-07-28:::2020-07-31:::2020-09-09:::2020-09-25:::2020-10-08:::2020-11-12"}], ["labeled_dependency", 0, 22, "1751158", {"sentence": "0", "chunk": "0", "entity": "Jamestown Invest 1, LLC", "relation": "company_id", "all_relations": ""}]]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legmapper_edgar_companyname| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|11.0 MB| ## References Manually scrapped Edgar Database --- layout: model title: XLNet Base CoNLL-03 NER Pipeline author: ahmedlone127 name: xlnet_base_token_classifier_conll03_pipeline date: 2022-06-14 tags: [ner, english, xlnet, base, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlnet_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/28/xlnet_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/xlnet_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655215928746.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/xlnet_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655215928746.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_posology_large_biobert_pipeline", "en", "clinical/models") text = '''The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_posology_large_biobert_pipeline", "en", "clinical/models") val text = "The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology_biobert_large.pipeline").predict("""The patient was prescribed 1 capsule of Advil 10 mg for 5 days and magnesium hydroxide 100mg/1ml suspension PO. He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals, and metformin 1000 mg two times a day.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------|--------:|------:|:------------|-------------:| | 0 | 1 | 27 | 27 | DOSAGE | 0.9998 | | 1 | capsule | 29 | 35 | FORM | 0.9978 | | 2 | Advil | 40 | 44 | DRUG | 0.9992 | | 3 | 10 mg | 46 | 50 | STRENGTH | 0.8269 | | 4 | for 5 days | 52 | 61 | DURATION | 0.978333 | | 5 | magnesium hydroxide | 67 | 85 | DRUG | 0.9783 | | 6 | 100mg/1ml | 87 | 95 | STRENGTH | 0.9336 | | 7 | suspension | 97 | 106 | FORM | 0.9999 | | 8 | PO | 108 | 109 | ROUTE | 0.9871 | | 9 | 40 units | 179 | 186 | DOSAGE | 0.6543 | | 10 | insulin glargine | 191 | 206 | DRUG | 0.97145 | | 11 | at night | 208 | 215 | FREQUENCY | 0.83505 | | 12 | 12 units | 218 | 225 | DOSAGE | 0.69795 | | 13 | insulin lispro | 230 | 243 | DRUG | 0.89265 | | 14 | with meals | 245 | 254 | FREQUENCY | 0.8772 | | 15 | metformin | 261 | 269 | DRUG | 1 | | 16 | 1000 mg | 271 | 277 | STRENGTH | 0.69955 | | 17 | two times a day | 279 | 293 | FREQUENCY | 0.758125 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_large_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal Payments Clause Binary Classifier author: John Snow Labs name: legclf_payments_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `payments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `payments` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_payments_clause_en_1.0.0_3.2_1660122833664.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_payments_clause_en_1.0.0_3.2_1660122833664.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_payments_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[payments]| |[other]| |[other]| |[payments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_payments_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.97 0.95 284 payments 0.91 0.84 0.87 105 accuracy - - 0.93 389 macro avg 0.92 0.90 0.91 389 weighted avg 0.93 0.93 0.93 389 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from jakobwes) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_squad_v1.1 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm_roberta_squad_v1.1` is a English model originally trained by `jakobwes`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_squad_v1.1_en_4.0.0_3.0_1655997017397.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_squad_v1.1_en_4.0.0_3.0_1655997017397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_squad_v1.1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_squad_v1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_squad_v1.1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|819.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jakobwes/xlm_roberta_squad_v1.1 --- layout: model title: Detect PHI for Deidentification (ner_deidentification_dl) author: John Snow Labs name: ner_deidentify_dl date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition annotator (NERDLModel) allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN.Deidentification NER (DL) is a Named Entity Recognition model that annotates text to find protected health information that may need to be deidentified. ## Predicted Entities `Age`,`BIOID`,`City`,`Country`,`Country`,`Date`,`Device`,`Doctor`,`EMail`,`Hospital`,`Fax`,`Healthplan`,`Hospital`,,`Idnum`,`Location-Other`,`Medicalrecord`,`Organization`,`Patient`,`Phone`,`Profession`,`State`,`Street`,`URL`,`Username`,`Zip` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_en_3.0.0_3.0_1617209710705.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_en_3.0.0_3.0_1617209710705.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained("ner_deidentify_dl","en","clinical/models") \ .setInputCols("sentence","token","embeddings") \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_deidentify_dl","en","clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid").predict("""A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street""") ```

## Results ```bash +---------------+-----+ |ner_label |count| +---------------+-----+ |O |28 | |I-HOSPITAL |4 | |B-DATE |3 | |I-STREET |3 | |I-PATIENT |2 | |B-DOCTOR |2 | |B-AGE |1 | |B-PATIENT |1 | |I-DOCTOR |1 | |B-MEDICALRECORD|1 | +---------------+-----+. +-----------------------------+-------------+ |chunk |ner_label | +-----------------------------+-------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson , Ora |PATIENT | |7194334 |MEDICALRECORD| |01/13/93 |DATE | |Oliveira |DOCTOR | |25 |AGE | |2079-11-09 |DATE | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street |STREET | +-----------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deidentify_dl| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on JSL enriched n2c2 2014: De-identification and Heart Disease Risk Factors Challenge datasets with embeddings_clinical https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/ ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|:-----------------|------:|-----:|-----:|---------:|---------:|---------:| | 1 | I-AGE | 7 | 3 | 6 | 0.7 | 0.538462 | 0.608696 | | 2 | I-DOCTOR | 800 | 27 | 94 | 0.967352 | 0.894855 | 0.929692 | | 3 | I-IDNUM | 6 | 0 | 2 | 1 | 0.75 | 0.857143 | | 4 | B-DATE | 1883 | 34 | 56 | 0.982264 | 0.971119 | 0.97666 | | 5 | I-DATE | 425 | 28 | 25 | 0.93819 | 0.944444 | 0.941307 | | 6 | B-PHONE | 29 | 7 | 9 | 0.805556 | 0.763158 | 0.783784 | | 7 | B-STATE | 87 | 4 | 11 | 0.956044 | 0.887755 | 0.920635 | | 8 | B-CITY | 35 | 11 | 26 | 0.76087 | 0.57377 | 0.654206 | | 9 | I-ORGANIZATION | 12 | 4 | 15 | 0.75 | 0.444444 | 0.55814 | | 10 | B-DOCTOR | 728 | 75 | 53 | 0.9066 | 0.932138 | 0.919192 | | 11 | I-PROFESSION | 43 | 11 | 13 | 0.796296 | 0.767857 | 0.781818 | | 12 | I-PHONE | 62 | 4 | 4 | 0.939394 | 0.939394 | 0.939394 | | 13 | B-AGE | 234 | 13 | 16 | 0.947368 | 0.936 | 0.94165 | | 14 | B-STREET | 20 | 7 | 16 | 0.740741 | 0.555556 | 0.634921 | | 15 | I-ZIP | 60 | 3 | 2 | 0.952381 | 0.967742 | 0.96 | | 16 | I-MEDICALRECORD | 54 | 5 | 2 | 0.915254 | 0.964286 | 0.93913 | | 17 | B-ZIP | 2 | 1 | 0 | 0.666667 | 1 | 0.8 | | 18 | B-HOSPITAL | 256 | 23 | 66 | 0.917563 | 0.795031 | 0.851913 | | 19 | I-STREET | 150 | 17 | 20 | 0.898204 | 0.882353 | 0.890208 | | 20 | B-COUNTRY | 22 | 2 | 8 | 0.916667 | 0.733333 | 0.814815 | | 21 | I-COUNTRY | 1 | 0 | 0 | 1 | 1 | 1 | | 22 | I-STATE | 6 | 0 | 1 | 1 | 0.857143 | 0.923077 | | 23 | B-USERNAME | 30 | 0 | 4 | 1 | 0.882353 | 0.9375 | | 24 | I-HOSPITAL | 295 | 37 | 64 | 0.888554 | 0.821727 | 0.853835 | | 25 | I-PATIENT | 243 | 26 | 41 | 0.903346 | 0.855634 | 0.878843 | | 26 | B-PROFESSION | 52 | 8 | 17 | 0.866667 | 0.753623 | 0.806202 | | 27 | B-IDNUM | 32 | 3 | 12 | 0.914286 | 0.727273 | 0.810127 | | 28 | I-CITY | 76 | 15 | 13 | 0.835165 | 0.853933 | 0.844444 | | 29 | B-PATIENT | 337 | 29 | 40 | 0.920765 | 0.893899 | 0.907133 | | 30 | B-MEDICALRECORD | 74 | 6 | 4 | 0.925 | 0.948718 | 0.936709 | | 31 | B-ORGANIZATION | 20 | 5 | 13 | 0.8 | 0.606061 | 0.689655 | | 32 | Macro-average | 6083 | 408 | 673 | 0.7976 | 0.697533 | 0.744218 | | 33 | Micro-average | 6083 | 408 | 673 | 0.937144 | 0.900385 | 0.918397 | ``` --- layout: model title: Google's Tapas Table Understanding (Small, WIKISQL) author: John Snow Labs name: table_qa_tapas_small_finetuned_wikisql_supervised date: 2022-09-30 tags: [en, table, qa, question, answering, open_source] task: Table Question Answering language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: TapasForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot Table Understanding Model which allows you to carry out Question Answering on Spark Dataframes. If you have a file stored in any table format, as csv, load it before using Spark. Size of this model: Small Has aggregation operations?: True ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/table_qa_tapas_small_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530735482.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/table_qa_tapas_small_finetuned_wikisql_supervised_en_4.2.0_3.0_1664530735482.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python json_data = """ { "header": ["name", "money", "age"], "rows": [ ["Donald Trump", "$100,000,000", "75"], ["Elon Musk", "$20,000,000,000,000", "55"] ] } """ queries = [ "Who earns less than 200,000,000?", "Who earns 100,000,000?", "How much money has Donald Trump?", "How old are they?", ] data = spark.createDataFrame([ [json_data, " ".join(queries)] ]).toDF("table_json", "questions") document_assembler = MultiDocumentAssembler() \ .setInputCols("table_json", "questions") \ .setOutputCols("document_table", "document_questions") sentence_detector = SentenceDetector() \ .setInputCols(["document_questions"]) \ .setOutputCol("questions") table_assembler = TableAssembler()\ .setInputCols(["document_table"])\ .setOutputCol("table") tapas = TapasForQuestionAnswering\ .pretrained("table_qa_tapas_small_finetuned_wikisql_supervised","en")\ .setInputCols(["questions", "table"])\ .setOutputCol("answers") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, table_assembler, tapas ]) model = pipeline.fit(data) model\ .transform(data)\ .selectExpr("explode(answers) AS answer")\ .select("answer")\ .show(truncate=False) ```

## Results ```bash +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |answer | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |{chunk, 0, 12, Donald Trump, {question -> Who earns less than 200,000,000?, aggregation -> NONE, cell_positions -> [0, 0], cell_scores -> 0.9999999}, []} | |{chunk, 0, 12, Donald Trump, {question -> Who earns 100,000,000?, aggregation -> NONE, cell_positions -> [0, 0], cell_scores -> 0.9999999}, []} | |{chunk, 0, 12, $100,000,000, {question -> How much money has Donald Trump?, aggregation -> NONE, cell_positions -> [1, 0], cell_scores -> 0.9999998}, []} | |{chunk, 0, 6, AVERAGE > 75, 55, {question -> How old are they?, aggregation -> AVERAGE, cell_positions -> [2, 0], [2, 1], cell_scores -> 0.99999976, 0.9999995}, []} | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|table_qa_tapas_small_finetuned_wikisql_supervised| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|110.1 MB| |Case sensitive:|false| ## References https://www.microsoft.com/en-us/download/details.aspx?id=54253 https://github.com/ppasupat/WikiTableQuestions https://github.com/salesforce/WikiSQL --- layout: model title: Legal Indemnity Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_indemnity_bert date: 2023-03-05 tags: [en, legal, classification, clauses, indemnity, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Indemnity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Indemnity`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_bert_en_1.0.0_3.0_1678049927751.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_bert_en_1.0.0_3.0_1678049927751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_indemnity_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Indemnity]| |[Other]| |[Other]| |[Indemnity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnity_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Indemnity 0.92 0.86 0.89 28 Other 0.91 0.95 0.93 43 accuracy - - 0.92 71 macro-avg 0.92 0.91 0.91 71 weighted-avg 0.92 0.92 0.91 71 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab3_by_sherry7144 TFWav2Vec2ForCTC from sherry7144 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab3_by_sherry7144 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab3_by_sherry7144` is a English model originally trained by sherry7144. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab3_by_sherry7144_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab3_by_sherry7144_en_4.2.0_3.0_1664041663422.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab3_by_sherry7144_en_4.2.0_3.0_1664041663422.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab3_by_sherry7144", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab3_by_sherry7144", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab3_by_sherry7144| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Legal Marketing Clause Binary Classifier author: John Snow Labs name: legclf_marketing_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `marketing` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `marketing` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_marketing_clause_en_1.0.0_3.2_1660122639196.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_marketing_clause_en_1.0.0_3.2_1660122639196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_marketing_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[marketing]| |[other]| |[other]| |[marketing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_marketing_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support marketing 0.50 0.50 0.50 2 other 0.93 0.93 0.93 15 accuracy - - 0.88 17 macro-avg 0.72 0.72 0.72 17 weighted-avg 0.88 0.88 0.88 17 ``` --- layout: model title: Translate English to Ruund Pipeline author: John Snow Labs name: translate_en_rnd date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, rnd, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `rnd` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_rnd_xx_2.7.0_2.4_1609688021276.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_rnd_xx_2.7.0_2.4_1609688021276.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_rnd", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_rnd", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.rnd').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_rnd| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_electric_pole_type_classification ViTForImageClassification from smc author: John Snow Labs name: image_classifier_vit_electric_pole_type_classification date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_electric_pole_type_classification` is a English model originally trained by smc. ## Predicted Entities `R300`, `Portico`, `R200`, `Tripode`, `H`, `Poste` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_electric_pole_type_classification_en_4.1.0_3.0_1660169367781.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_electric_pole_type_classification_en_4.1.0_3.0_1660169367781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_electric_pole_type_classification", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_electric_pole_type_classification", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_electric_pole_type_classification| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Stopwords Remover for Japanese language (153 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ja, open_source] task: Stop Words Removal language: ja edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ja_3.4.1_3.0_1646672369133.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ja_3.4.1_3.0_1646672369133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------+ |result | +--------------------------------+ |[あなたは私よりも優れていません]| +--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ja| |Size:|1.8 KB| --- layout: model title: Clinical Deidentification Pipeline (English) author: John Snow Labs name: clinical_deidentification date: 2022-09-14 tags: [deidendification, deid, en, licensed, clinical, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_4.1.0_3.2_1663183687291.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_4.1.0_3.2_1663183687291.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification","en","clinical/models") val sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.clinical_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

## Results ```bash Masked with entity labels ------------------------------ Name : , Record date: , # . Dr. , ID, IP . He is a male was admitted to the for cystectomy on . Patient's VIN : , SSN , Driver's license . Phone , , , E-MAIL: . Masked with chars ------------------------------ Name : [**************], Record date: [********], # [****]. Dr. [********], ID[**********], IP [************]. He is a [*********] male was admitted to the [**********] for cystectomy on [******]. Patient's VIN : [***************], SSN [**********], Driver's license [*********]. Phone [************], [***************], [***********], E-MAIL: [*************]. Masked with fixed length chars ------------------------------ Name : ****, Record date: ****, # ****. Dr. ****, ID****, IP ****. He is a **** male was admitted to the **** for cystectomy on ****. Patient's VIN : ****, SSN ****, Driver's license ****. Phone ****, ****, ****, E-MAIL: ****. Obfuscated ------------------------------ Name : Craige Perks, Record date: 2093-02-06, # R2593192. Dr. Dr Felice Lacer, IDXO:4884578, IP 444.444.444.444. He is a 75 male was admitted to the MADISON VALLEY MEDICAL CENTER for cystectomy on 07-01-1972. Patient's VIN : 2BBBB11BBBB222999, SSN SSN-814-86-1962, Driver's license P055567317431. Phone 0381-6762484, Budaörsi út 14., New brunswick, E-MAIL: Reba@google.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - TextMatcherModel - ContextualParserModel - RegexMatcherModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English BertForMaskedLM Base Cased model (from model-attribution-challenge) author: John Snow Labs name: bert_embeddings_model_attribution_challenge_base_cased date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased` is a English model originally trained by `model-attribution-challenge`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_model_attribution_challenge_base_cased_en_4.2.4_3.0_1670016326907.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_model_attribution_challenge_base_cased_en_4.2.4_3.0_1670016326907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_model_attribution_challenge_base_cased","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_model_attribution_challenge_base_cased","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_model_attribution_challenge_base_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/model-attribution-challenge/bert-base-cased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: Translate English to Nyaneka Pipeline author: John Snow Labs name: translate_en_nyk date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, nyk, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `nyk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_nyk_xx_2.7.0_2.4_1609689594225.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_nyk_xx_2.7.0_2.4_1609689594225.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_nyk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_nyk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.nyk').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_nyk| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Legal proceedings Clause Binary Classifier author: John Snow Labs name: legclf_legal_proceedings_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `legal-proceedings` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `legal-proceedings` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_legal_proceedings_clause_en_1.0.0_3.2_1660122602093.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_legal_proceedings_clause_en_1.0.0_3.2_1660122602093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_legal_proceedings_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[legal-proceedings]| |[other]| |[other]| |[legal-proceedings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_legal_proceedings_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support legal-proceedings 0.96 0.79 0.87 29 other 0.95 0.99 0.97 106 accuracy - - 0.95 135 macro-avg 0.95 0.89 0.92 135 weighted-avg 0.95 0.95 0.95 135 ``` --- layout: model title: English asr_wav2vec2_large_robust_LS960 TFWav2Vec2ForCTC from leonardvorbeck author: John Snow Labs name: asr_wav2vec2_large_robust_LS960 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_LS960` is a English model originally trained by leonardvorbeck. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_robust_LS960_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_LS960_en_4.2.0_3.0_1664023686921.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_robust_LS960_en_4.2.0_3.0_1664023686921.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_robust_LS960", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_robust_LS960", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_robust_LS960| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|757.6 MB| --- layout: model title: Abkhazian asr_xls_r_eng TFWav2Vec2ForCTC from mattchurgin author: John Snow Labs name: asr_xls_r_eng date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_eng` is a Abkhazian model originally trained by mattchurgin. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_eng_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_eng_ab_4.2.0_3.0_1664019618400.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_eng_ab_4.2.0_3.0_1664019618400.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_eng", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_eng", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_eng| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|1.4 MB| --- layout: model title: Turkish BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_tr_cased date: 2022-12-02 tags: [tr, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: tr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-tr-cased` is a Turkish model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_tr_cased_tr_4.2.4_3.0_1670019105736.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_tr_cased_tr_4.2.4_3.0_1670019105736.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_tr_cased","tr") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_tr_cased","tr") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_tr_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|tr| |Size:|378.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-tr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from jdang) author: John Snow Labs name: xlmroberta_ner_jdang_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `jdang`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jdang_base_finetuned_panx_de_4.1.0_3.0_1660434450928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jdang_base_finetuned_panx_de_4.1.0_3.0_1660434450928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jdang_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jdang_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jdang_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jdang/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from sam999) author: John Snow Labs name: distilbert_qa_sam999_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `sam999`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sam999_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772406690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sam999_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772406690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sam999_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sam999_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sam999_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sam999/distilbert-base-uncased-finetuned-squad --- layout: model title: Explain Document DL author: John Snow Labs name: explain_document_dl date: 2020-03-19 task: [Sentence Detection, Part of Speech Tagging, Lemmatization, Pipeline Public, Spell Check] language: en nav_key: models edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [pipeline, en, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description The ``explain_document_dl`` is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_dl_en_2.4.3_2.4_1584626657780.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_dl_en_2.4.3_2.4_1584626657780.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('explain_document_dl', lang = 'en') annotations = pipeline.fullAnnotate("""French author who helped pioner the science-fiction genre. Verne wrate about space, air, and underwater travel before navigable aircrast and practical submarines were invented, and before any means of space travel had been devised.""")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline('explain_document_dl', lang = 'en') val result = pipeline.fullAnnotate("French author who helped pioner the science-fiction genre. Verne wrate about space, air, and underwater travel before navigable aircrast and practical submarines were invented, and before any means of space travel had been devised.")(0) ``` {:.nlu-block} ```python import nlu text = ["""John Snow built a detailed map of all the households where people died, and came to the conclusion that the fault was one public water pump that all the victims had used."""] explain_df = nlu.load('en.explain.dl').predict(text) explain_df ```

{:.h2_title} ## Results {:.result_box} ```bash +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ | text| document| sentence| token| spell| lemmas| stems| pos| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ |French author who...|[[document, 0, 23...|[[document, 0, 57...|[[token, 0, 5, Fr...|[[token, 0, 5, Fr...|[[token, 0, 5, Fr...|[[token, 0, 5, fr...|[[pos, 0, 5, JJ, ...| +--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_dl| |Type:|pipeline| |Compatibility:|Spark NLP 2.5.5+| |License:|Open Source| |Edition:|Community| |Language:|[en]| {:.h2_title} ## Included Models The explain_document_dl has one Transformer and six annotators: - Documenssembler - A Transformer that creates a column that contains documents. - Sentence Segmenter - An annotator that produces the sentences of the document. - Tokenizer - An annotator that produces the tokens of the sentences. - SpellChecker - An annotator that produces the spelling-corrected tokens. - Stemmer - An annotator that produces the stems of the tokens. - Lemmatizer - An annotator that produces the lemmas of the tokens. - POS Tagger - An annotator that produces the parts of speech of the associated tokens. --- layout: model title: Pipeline to Detect Neoplasms author: John Snow Labs name: ner_neoplasms_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, es] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_neoplasms](https://nlp.johnsnowlabs.com/2021/03/31/ner_neoplasms_es.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_neoplasms_pipeline_es_4.3.0_3.2_1678879327865.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_neoplasms_pipeline_es_4.3.0_3.2_1678879327865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_neoplasms_pipeline", "es", "clinical/models") text = '''HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_neoplasms_pipeline", "es", "clinical/models") val text = "HISTORIA DE ENFERMEDAD ACTUAL: El Sr. Smith es un hombre blanco veterano de 60 años con múltiples comorbilidades, que tiene antecedentes de cáncer de vejiga diagnosticado hace aproximadamente dos años por el Hospital VA. Allí se sometió a una resección. Debía ser ingresado en el Hospital de Día para una cistectomía. Fue visto en la Clínica de Urología y Clínica de Radiología el 02/04/2003. CURSO DE HOSPITAL: El Sr. Smith se presentó en el Hospital de Día antes de la cirugía de Urología. En evaluación, EKG, ecocardiograma fue anormal, se obtuvo una consulta de Cardiología. Luego se procedió a una resonancia magnética de estrés con adenosina cardíaca, la misma fue positiva para isquemia inducible, infarto subendocárdico inferolateral leve a moderado con isquemia peri-infarto. Además, se observa isquemia inducible en el tabique lateral inferior. El Sr. Smith se sometió a un cateterismo del corazón izquierdo, que reveló una enfermedad de las arterias coronarias de dos vasos. La RCA, proximal estaba estenosada en un 95% y la distal en un 80% estenosada. La LAD media estaba estenosada en un 85% y la LAD distal estaba estenosada en un 85%. Se colocaron cuatro stents de metal desnudo Multi-Link Vision para disminuir las cuatro lesiones al 0%. Después de la intervención, el Sr. Smith fue admitido en 7 Ardmore Tower bajo el Servicio de Cardiología bajo la dirección del Dr. Hart. El Sr. Smith tuvo un curso hospitalario post-intervención sin complicaciones. Se mantuvo estable para el alta hospitalaria el 07/02/2003 con instrucciones de tomar Plavix diariamente durante un mes y Urología está al tanto de lo mismo." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------|--------:|------:|:---------------------|-------------:| | 0 | cáncer | 140 | 145 | MORFOLOGIA_NEOPLASIA | 0.9997 | | 1 | Multi-Link | 1195 | 1204 | MORFOLOGIA_NEOPLASIA | 0.574 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_neoplasms_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|383.2 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Legal Relation Extraction (Whereas, sm, Bidirectional)) author: John Snow Labs name: legre_whereas date: 2022-08-24 tags: [en, legal, re, relations, licensed] task: Relation Extraction language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_cuad_whereas_clause` Text Classifier to select only these paragraphs; This is a Relation Extraction model to infer relations between elements in WHEREAS clauses, more specifically the SUBJECT, the ACTION and the OBJECT. There are two relations possible: `has_subject` and `has_object`. You can also use `legpipe_whereas` which includes this model and its NER and also depedency parsing, to carry out chunk extraction using grammatical features (the dependency tree). This model is a `sm` model without meaningful directions in the relations (the model was not trained to understand if the direction of the relation is from left to right or right to left). There are bigger models in Models Hub trained also with directed relationships. ## Predicted Entities `has_subject`, `has_object` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legre_whereas_en_1.0.0_3.2_1661341573628.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legre_whereas_en_1.0.0_3.2_1661341573628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = legal.NerModel.pretrained('legner_whereas', 'en', 'legal/models')\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") reDL = legal.RelationExtractionDLModel\ .pretrained("legre_whereas", "en", "legal/models")\ .setPredictionThreshold(0.5)\ .setInputCols(["ner_chunk", "document"])\ .setOutputCol("relations") pipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner_model, ner_converter, reDL ]) text = """ WHEREAS VerticalNet owns and operates a series of online communities ( as defined below ) that are accessible via the world wide web , each of which is designed to be an online gathering place for businesses of a certain type or within a certain industry ; """ data = spark.createDataFrame([[text]]).toDF("text") model = pipeline.fit(data) res = model.transform(data) ```

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence has_subject WHEREAS_SUBJECT 11 21 VerticalNet WHEREAS_ACTION 32 39 operates 0.9982886 has_subject WHEREAS_SUBJECT 11 21 VerticalNet WHEREAS_OBJECT 41 70 a series of online communities 0.9890683 has_object WHEREAS_ACTION 32 39 operates WHEREAS_OBJECT 41 70 a series of online communities 0.7831568 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legre_whereas| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|409.9 MB| ## References Manual annotations on CUAD dataset ## Benchmarking ```bash label Recall Precision F1 Support has_object 0.946 0.981 0.964 56 has_subject 0.952 0.988 0.969 83 no_rel 1.000 0.970 0.985 161 Avg. 0.966 0.980 0.973 - Weighted-Avg. 0.977 0.977 0.977 - ``` --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_orkg_how_5e_05 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-orkg-how-5e-05` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_how_5e_05_en_4.3.0_3.0_1672766726781.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_how_5e_05_en_4.3.0_3.0_1672766726781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_how_5e_05","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_how_5e_05","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_orkg_how_5e_05| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Moussab/distilbert-base-cased-distilled-squad-orkg-how-5e-05 --- layout: model title: English AlbertForQuestionAnswering model (from nlpunibo) author: John Snow Labs name: albert_qa_nlpunibo date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `albert` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_nlpunibo_en_4.0.0_3.0_1656063694630.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_nlpunibo_en_4.0.0_3.0_1656063694630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_nlpunibo","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_nlpunibo","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.albert.by_nlpunibo").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_nlpunibo| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/albert --- layout: model title: English asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent TFWav2Vec2ForCTC from creynier author: John Snow Labs name: asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent` is a English model originally trained by creynier. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_en_4.2.0_3.0_1664042438490.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent_en_4.2.0_3.0_1664042438490.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_swbd_turn_eos_long_short_utt_removed_5percent| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|349.4 MB| --- layout: model title: Detect Living Species (w2v_cc_300d) author: John Snow Labs name: ner_living_species date: 2022-06-23 tags: [it, ner, clinical, licensed] task: Named Entity Recognition language: it edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Italian which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `w2v_cc_300d` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_it_3.5.3_3.0_1655972680773.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_it_3.5.3_3.0_1655972680773.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "it")\ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species", "it", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "it") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species", "it", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("it.med_ner.living_species").predict("""Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.""") ```

## Results ```bash +----------------+-------+ |ner_chunk |label | +----------------+-------+ |donna |HUMAN | |personale |HUMAN | |madre |HUMAN | |fratello |HUMAN | |sorella |HUMAN | |virus epatotropi|SPECIES| |HBV |SPECIES| |HCV |SPECIES| |HIV |SPECIES| |paziente |HUMAN | +----------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|it| |Size:|15.1 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.83 0.97 0.89 2772 B-SPECIES 0.60 0.92 0.73 2866 I-HUMAN 0.67 0.61 0.64 101 I-SPECIES 0.56 0.87 0.68 1039 micro-avg 0.68 0.93 0.78 6778 macro-avg 0.67 0.84 0.74 6778 weighted-avg 0.69 0.93 0.79 6778 ``` --- layout: model title: Spanish RoBERTa Embeddings (Bertin Base) author: John Snow Labs name: roberta_embeddings_bertin_roberta_base_spanish date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model for Spanish Language, trained within the Bertin project. Other non-base Bertin models can be found [here](https://nlp.johnsnowlabs.com/models?q=bertin). The model was uploaded to Hugging Face, adapted and imported into Spark NLP. `bertin-roberta-base-spanish` is a Spanish model orginally trained by `bertin-project`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_roberta_base_spanish_es_3.4.2_3.0_1649945200032.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_roberta_base_spanish_es_3.4.2_3.0_1649945200032.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bertin_roberta_base_spanish| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|234.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/bertin-project/bertin-roberta-base-spanish - https://github.com/google/flax - https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers - https://arxiv.org/pdf/2107.07253.pdf - https://arxiv.org/abs/1907.11692 - https://www.wired.com/2017/04/courts-using-ai-sentence-criminals-must-stop-now/ - https://www.washingtonpost.com/technology/2019/05/16/police-have-used-celebrity-lookalikes-distorted-images-boost-facial-recognition-results-research-finds/ - https://www.wired.com/story/ai-college-exam-proctors-surveillance/ - https://www.eff.org/deeplinks/2020/09/students-are-pushing-back-against-proctoring-surveillance-apps - https://www.washingtonpost.com/technology/2019/10/22/ai-hiring-face-scanning-algorithm-increasingly-decides-whether-you-deserve-job/ - https://www.technologyreview.com/2021/07/21/1029860/disability-rights-employment-discrimination-ai-hiring/ - https://www.insider.com/china-is-testing-ai-recognition-on-the-uighurs-bbc-2021-5 - https://www.health.harvard.edu/blog/anti-asian-racism-breaking-through-stereotypes-and-silence-2021041522414 - https://discord.com/channels/858019234139602994/859113060068229190 --- layout: model title: Part of Speech for Indonesian author: John Snow Labs name: pos_ud_gsd date: 2021-03-09 tags: [part_of_speech, open_source, indonesian, pos_ud_gsd, id] task: Part of Speech Tagging language: id edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - PROPN - AUX - DET - NOUN - PRON - VERB - ADP - PUNCT - ADV - CCONJ - SCONJ - NUM - ADJ - PART - SYM - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_id_3.0.0_3.0_1615292193447.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_id_3.0.0_3.0_1615292193447.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_gsd", "id") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Halo dari John Salju Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_gsd", "id") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Halo dari John Salju Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Halo dari John Salju Labs! ""] token_df = nlu.load('id.pos').predict(text) token_df ```

## Results ```bash token pos 0 Halo PROPN 1 dari ADP 2 John PROPN 3 Salju PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|id| --- layout: model title: Arabic Bert Embeddings (Base, MSA dataset) author: John Snow Labs name: bert_embeddings_bert_base_arabic_camelbert_msa date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-msa` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_ar_3.4.2_3.0_1649678312712.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_ar_3.4.2_3.0_1649678312712.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabic_camelbert_msa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: German asr_wav2vec2_base_german_cv9 TFWav2Vec2ForCTC from oliverguhr author: John Snow Labs name: asr_wav2vec2_base_german_cv9 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_german_cv9` is a German model originally trained by oliverguhr. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_german_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_german_cv9_de_4.2.0_3.0_1664101206544.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_german_cv9_de_4.2.0_3.0_1664101206544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_german_cv9", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_german_cv9", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_german_cv9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|349.2 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Isoko author: John Snow Labs name: opus_mt_en_iso date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, iso, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `iso` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_iso_xx_2.7.0_2.4_1609167424385.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_iso_xx_2.7.0_2.4_1609167424385.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_iso| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Armenian to English Pipeline author: John Snow Labs name: translate_hy_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, hy, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `hy` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_hy_en_xx_2.7.0_2.4_1609691902821.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_hy_en_xx_2.7.0_2.4_1609691902821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_hy_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_hy_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.hy.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_hy_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal No conflicts Clause Binary Classifier author: John Snow Labs name: legclf_no_conflicts_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-conflicts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-conflicts` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_conflicts_clause_en_1.0.0_3.2_1660122683948.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_conflicts_clause_en_1.0.0_3.2_1660122683948.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_conflicts_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-conflicts]| |[other]| |[other]| |[no-conflicts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_conflicts_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-conflicts 0.96 0.94 0.95 54 other 0.98 0.99 0.98 153 accuracy - - 0.98 207 macro-avg 0.97 0.97 0.97 207 weighted-avg 0.98 0.98 0.98 207 ``` --- layout: model title: Part of Speech for Hungarian author: John Snow Labs name: pos_ud_szeged date: 2020-05-05 12:50:00 +0800 task: Part of Speech Tagging language: hu edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, hu] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_szeged_hu_2.5.0_2.4_1588671966774.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_szeged_hu_2.5.0_2.4_1588671966774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_szeged", "hu") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Az északi király kivételével John Snow angol orvos, vezető szerepet játszik az érzéstelenítés és az orvosi higiénia fejlesztésében.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_szeged", "hu") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Az északi király kivételével John Snow angol orvos, vezető szerepet játszik az érzéstelenítés és az orvosi higiénia fejlesztésében.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Az északi király kivételével John Snow angol orvos, vezető szerepet játszik az érzéstelenítés és az orvosi higiénia fejlesztésében."""] pos_df = nlu.load('hu.pos.ud_szeged').predict(text) pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=1, result='DET', metadata={'word': 'Az'}), Row(annotatorType='pos', begin=3, end=8, result='ADJ', metadata={'word': 'északi'}), Row(annotatorType='pos', begin=10, end=15, result='NOUN', metadata={'word': 'király'}), Row(annotatorType='pos', begin=17, end=27, result='NOUN', metadata={'word': 'kivételével'}), Row(annotatorType='pos', begin=29, end=32, result='PROPN', metadata={'word': 'John'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_szeged| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|hu| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Self Report Classification Pipeline - Voice of the Patient author: John Snow Labs name: bert_sequence_classifier_vop_self_report_pipeline date: 2023-06-14 tags: [licensed, en, clinical, vop, classification] task: Text Classification language: en edition: Healthcare NLP 4.4.3 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline includes the Medical Bert for Sequence Classification model to classify texts depending on if they are self-reported or if they refer to another person. The pipeline is built on the top of [bert_sequence_classifier_vop_self_report](https://nlp.johnsnowlabs.com/2023/06/13/bert_sequence_classifier_vop_self_report_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_self_report_pipeline_en_4.4.3_3.2_1686702483761.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_self_report_pipeline_en_4.4.3_3.2_1686702483761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | text | prediction | |:---------------------------------------------------------|:-------------| | My friend was treated for her skin cancer two years ago. | 3rd_Person | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_self_report_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|406.4 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForSequenceClassification --- layout: model title: English Legal RobertaForMaskedLM Base Cased model author: John Snow Labs name: roberta_embeddings_legal_base date: 2022-12-12 tags: [en, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-roberta-base` is a English model originally trained by `saibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_legal_base_en_4.2.4_3.0_1670858851348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_legal_base_en_4.2.4_3.0_1670858851348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_legal_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_legal_base","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_legal_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|468.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/saibo/legal-roberta-base - https://www.kaggle.com/uspto/patent-litigations - https://case.law/ - https://www.kaggle.com/bigquery/patents - https://www.kaggle.com/sohier/beyond-queries-exploring-the-bigquery-api --- layout: model title: English BertForQuestionAnswering model (from machine2049) author: John Snow Labs name: bert_qa_bert_base_uncased_finetuned_duorc_bert date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-duorc_bert` is a English model orginally trained by `machine2049`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_duorc_bert_en_4.0.0_3.0_1654180931013.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_finetuned_duorc_bert_en_4.0.0_3.0_1654180931013.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_finetuned_duorc_bert","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_finetuned_duorc_bert","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.base_uncased.by_machine2049").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_finetuned_duorc_bert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/machine2049/bert-base-uncased-finetuned-duorc_bert --- layout: model title: Context Spell Checker Pipeline for English author: John Snow Labs name: spellcheck_dl_pipeline date: 2022-04-14 tags: [spellcheck, spell, spellcheck_pipeline, spelling_corrector, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained spellchecker pipeline is built on the top of [spellcheck_dl](https://nlp.johnsnowlabs.com/2022/04/01/spellcheck_dl_en_2_4.html) model. This pipeline is for PySpark 2.4.x users with SparkNLP 3.4.1. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.1_2.4_1649941123093.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_pipeline_en_3.4.1_2.4_1649941123093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash [{'checked': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'weather', '.'], 'document': ['During the summer we have the best ueather.'], 'token': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'ueather', '.']}, {'checked': ['I', 'have', 'a', 'black', 'leather', 'jacket', ',', 'so', 'nice', '.'], 'document': ['I have a black ueather jacket, so nice.'], 'token': ['I', 'have', 'a', 'black', 'ueather', 'jacket', ',', 'so', 'nice', '.']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_dl_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.1| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|99.7 MB| ## Included Models - DocumentAssembler - TokenizerModel - ContextSpellCheckerModel --- layout: model title: Legal Industrial Structures And Policy Document Classifier (EURLEX) author: John Snow Labs name: legclf_industrial_structures_and_policy_bert date: 2023-03-06 tags: [en, legal, classification, clauses, industrial_structures_and_policy, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_industrial_structures_and_policy_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Industrial_Structures_and_Policy or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Industrial_Structures_and_Policy`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_industrial_structures_and_policy_bert_en_1.0.0_3.0_1678111630461.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_industrial_structures_and_policy_bert_en_1.0.0_3.0_1678111630461.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_industrial_structures_and_policy_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Industrial_Structures_and_Policy]| |[Other]| |[Other]| |[Industrial_Structures_and_Policy]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_industrial_structures_and_policy_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Industrial_Structures_and_Policy 0.80 0.80 0.80 40 Other 0.84 0.84 0.84 51 accuracy - - 0.82 91 macro-avg 0.82 0.82 0.82 91 weighted-avg 0.82 0.82 0.82 91 ``` --- layout: model title: Tamil Bert Embeddings author: John Snow Labs name: bert_embeddings_muril_adapted_local date: 2022-04-11 tags: [bert, embeddings, ta, open_source] task: Embeddings language: ta edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `muril-adapted-local` is a Tamil model orginally trained by `monsoon-nlp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_ta_3.4.2_3.0_1649676437079.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_muril_adapted_local_ta_3.4.2_3.0_1649676437079.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_muril_adapted_local| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ta| |Size:|888.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/monsoon-nlp/muril-adapted-local - https://tfhub.dev/google/MuRIL/1 --- layout: model title: Smaller BERT Sentence Embeddings (L-6_H-512_A-8) author: John Snow Labs name: sent_small_bert_L6_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L6_512_en_2.6.0_2.4_1598350624049.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L6_512_en_2.6.0_2.4_1598350624049.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L6_512", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L6_512", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L6_512').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_small_bert_L6_512_embeddings I hate cancer [0.07266588509082794, 1.1798694133758545, -0.1... Antibiotics aren't painkiller [0.2663014829158783, 1.0382004976272583, -0.41... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L6_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-256-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2_en_4.3.0_3.0_1674214836583.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2_en_4.3.0_3.0_1674214836583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_256_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|427.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-256-finetuned-squad-seed-2 --- layout: model title: Chinese Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_jdt_fin_roberta_wwm_large date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `jdt-fin-roberta-wwm-large` is a Chinese model orginally trained by `wangfan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_jdt_fin_roberta_wwm_large_zh_3.4.2_3.0_1649670973199.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_jdt_fin_roberta_wwm_large_zh_3.4.2_3.0_1649670973199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_jdt_fin_roberta_wwm_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/wangfan/jdt-fin-roberta-wwm-large --- layout: model title: Extract Treatment Entities from Voice of the Patient Documents (embeddings_clinical) author: John Snow Labs name: ner_vop_treatment date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, treatment, drug] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts treatments mentioned in documents transferred from the patient’s own sentences. ## Predicted Entities `Drug`, `Form`, `Dosage`, `Frequency`, `Route`, `Duration`, `Procedure`, `Treatment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_en_4.4.3_3.0_1686077083722.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_en_4.4.3_3.0_1686077083722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------|:------------| | metformin | Drug | | glipizide | Drug | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_treatment| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Drug 1297 107 143 1440 0.92 0.90 0.91 Form 246 36 20 266 0.87 0.92 0.90 Dosage 346 53 66 412 0.87 0.84 0.85 Frequency 909 185 170 1079 0.83 0.84 0.84 Route 38 5 10 48 0.88 0.79 0.84 Duration 1825 252 485 2310 0.88 0.79 0.83 Procedure 565 101 140 705 0.85 0.80 0.82 Treatment 144 25 84 228 0.85 0.63 0.73 macro_avg 5370 764 1118 6488 0.87 0.81 0.84 micro_avg 5370 764 1118 6488 0.88 0.83 0.85 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from jsunster) author: John Snow Labs name: distilbert_qa_jsunster_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `jsunster`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_jsunster_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771511693.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_jsunster_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771511693.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_jsunster_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_jsunster_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_jsunster_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/jsunster/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from LenaSchmidt) author: John Snow Labs name: distilbert_qa_lenaschmidt_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `LenaSchmidt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_lenaschmidt_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768620901.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_lenaschmidt_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768620901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lenaschmidt_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_lenaschmidt_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_lenaschmidt_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/LenaSchmidt/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Detect Chemical Compounds and Genes author: John Snow Labs name: bert_token_classifier_ner_chemprot_pipeline date: 2022-03-15 tags: [chemprot, bert_token_classifier, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemprot](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemprot_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMPROT_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CHEMPROT_CLINICAL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_3.0_1647339959529.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_3.0_1647339959529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab0_by_tahazakir", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab0_by_tahazakir", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab0_by_tahazakir| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534_de_4.2.0_3.0_1664105023083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534_de_4.2.0_3.0_1664105023083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s534| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English asr_wav2vec2_base_checkpoint_9 TFWav2Vec2ForCTC from jiobiala24 author: John Snow Labs name: pipeline_asr_wav2vec2_base_checkpoint_9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_checkpoint_9` is a English model originally trained by jiobiala24. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_checkpoint_9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_checkpoint_9_en_4.2.0_3.0_1664021052269.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_checkpoint_9_en_4.2.0_3.0_1664021052269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_checkpoint_9', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_checkpoint_9", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_checkpoint_9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.2 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_wav2vec2_base_960h_4_gram TFWav2Vec2ForCTC from patrickvonplaten author: John Snow Labs name: pipeline_asr_wav2vec2_base_960h_4_gram date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_960h_4_gram` is a English model originally trained by patrickvonplaten. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_960h_4_gram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_960h_4_gram_en_4.2.0_3.0_1664022212611.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_960h_4_gram_en_4.2.0_3.0_1664022212611.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_960h_4_gram', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_960h_4_gram", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_960h_4_gram| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.6 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Indic Languages author: John Snow Labs name: opus_mt_en_inc date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, inc, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `inc` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_inc_xx_2.7.0_2.4_1609167901916.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_inc_xx_2.7.0_2.4_1609167901916.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_inc| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Chunk Entity Resolver for ICD10 codes author: John Snow Labs name: chunkresolve_ICD10GM_2021 date: 2021-04-16 tags: [entity_resolution, clinical, licensed, de] task: Entity Resolution language: de edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-GM codes for German language using chunk embeddings (augmented with synonyms, four times richer than previous resolver). ## Predicted Entities ICD10 codes {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_GM_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_ICD10GM_2021_de_3.0.0_3.0_1618603791008.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_ICD10GM_2021_de_3.0.0_3.0_1618603791008.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... resolver = ChunkEntityResolverModel.pretrained("chunkresolve_ICD10GM_2021","de","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") pipeline = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, resolver]) data = spark.createDataFrame([["metastatic lung cancer"]]).toDF("text") model = pipeline.fit(data) results = model.transform(data) ... ``` ```scala ... val resolver = ChunkEntityResolverModel.pretrained("chunkresolve_ICD10GM_2021","de","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(document_assembler, sbert_embedder, resolver)) val data = Seq("metastatic lung cancer").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_ICD10GM_2021| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[recognized]| |Language:|de| --- layout: model title: Stopwords Remover for Icelandic language (152 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, is, open_source] task: Stop Words Removal language: is edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_is_3.4.1_3.0_1646673153470.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_is_3.4.1_3.0_1646673153470.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------+ |result | +------------------------+ |[Þú, ert, betri, en, ég]| +------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|is| |Size:|1.9 KB| --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_1_impartit_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `set_date_1-impartit_4-bert` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_impartit_4_en_4.0.0_3.0_1657191771743.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_impartit_4_en_4.0.0_3.0_1657191771743.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_impartit_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_impartit_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_1_impartit_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/set_date_1-impartit_4-bert --- layout: model title: English image_classifier_vit_animals_classifier ViTForImageClassification from victor author: John Snow Labs name: image_classifier_vit_animals_classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_animals_classifier` is a English model originally trained by victor. ## Predicted Entities `lion`, `dolphin`, `hippo`, `giraffe`, `elephant` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animals_classifier_en_4.1.0_3.0_1660168625857.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animals_classifier_en_4.1.0_3.0_1660168625857.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_animals_classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_animals_classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_animals_classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Pipeline to Detect Radiology Entities, Assign Assertion Status and Find Relations author: John Snow Labs name: explain_clinical_doc_radiology date: 2022-03-31 tags: [licensed, clinical, en, ner, assertion, relation_extraction, radiology] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline for detecting radiology entities with the `ner_radiology` NER model, assigning their assertion status with `assertion_dl_radiology` model, and extracting relations between the diagnosis, test, and findings with `re_test_problem_finding` relation extraction model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_radiology_en_3.4.2_3.0_1648737971620.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_radiology_en_3.4.2_3.0_1648737971620.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("explain_clinical_doc_radiology", "en", "clinical/models") result = pipeline.fullAnnotate("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""")[0] ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("explain_clinical_doc_radiology", "en", "clinical/models") val result = pipeline.fullAnnotate("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""")(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.clinical_radiology.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash +----+------------------------------------------+---------------------------+ | | chunks | entities | |---:|:-----------------------------------------|:--------------------------| | 0 | Bilateral breast | BodyPart | | 1 | ultrasound | ImagingTest | | 2 | ovoid mass | ImagingFindings | | 3 | 0.5 x 0.5 x 0.4 | Measurements | | 4 | cm | Units | | 5 | anteromedial aspect of the left shoulder | BodyPart | | 6 | mass | ImagingFindings | | 7 | isoechoic echotexture | ImagingFindings | | 8 | muscle | BodyPart | | 9 | internal color flow | ImagingFindings | | 10 | benign fibrous tissue | ImagingFindings | | 11 | lipoma | Disease_Syndrome_Disorder | +----+------------------------------------------+---------------------------+ +----+-----------------------+---------------------------+-------------+ | | chunks | entities | assertion | |---:|:----------------------|:--------------------------|:------------| | 0 | ultrasound | ImagingTest | Confirmed | | 1 | ovoid mass | ImagingFindings | Confirmed | | 2 | mass | ImagingFindings | Confirmed | | 3 | isoechoic echotexture | ImagingFindings | Confirmed | | 4 | internal color flow | ImagingFindings | Negative | | 5 | benign fibrous tissue | ImagingFindings | Suspected | | 6 | lipoma | Disease_Syndrome_Disorder | Suspected | +----+-----------------------+---------------------------+-------------+ +---------+-----------------+-----------------------+---------------------------+------------+ |relation | entity1 | chunk1 | entity2 | chunk2 | |--------:|:----------------|:----------------------|:--------------------------|:-----------| | 1 | ImagingTest | ultrasound | ImagingFindings | ovoid mass | | 0 | ImagingFindings | benign fibrous tissue | Disease_Syndrome_Disorder | lipoma | +---------+-----------------+-----------------------+---------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_radiology| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternal - NerConverterInternal - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel --- layout: model title: Pipeline to Detect Anatomical Structures (Single Entity - embeddings_clinical) author: John Snow Labs name: ner_anatomy_coarse_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_anatomy_coarse](https://nlp.johnsnowlabs.com/2021/03/31/ner_anatomy_coarse_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_pipeline_en_4.3.0_3.2_1678862662568.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_anatomy_coarse_pipeline_en_4.3.0_3.2_1678862662568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_anatomy_coarse_pipeline", "en", "clinical/models") text = '''content in the lung tissue''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_anatomy_coarse_pipeline", "en", "clinical/models") val text = "content in the lung tissue" val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.anatomy_coarse.pipeline").predict("""content in the lung tissue""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:------------|-------------:| | 0 | lung tissue | 15 | 25 | Anatomy | 0.99655 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_anatomy_coarse_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Detect Bacterial Species (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_bacteria date: 2021-09-30 tags: [bacteria, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.2.2 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect different types of species of bacteria in text using pretrained NER model. This model is trained with the `BertForTokenClassification` method from `transformers` library and imported into Spark NLP. ## Predicted Entities `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_en_3.2.2_2.4_1632995062374.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_en_3.2.2_2.4_1632995062374.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |SMSP (T) |SPECIES | |Methanoregula formicica|SPECIES | |SMSP (T) |SPECIES | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bacteria| |Compatibility:|Healthcare NLP 3.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|512| ## Data Source Trained on a custom dataset by John Snow Labs. ## Benchmarking ```bash label precision recall f1-score support B-SPECIES 0.98 0.84 0.91 767 I-SPECIES 0.99 0.84 0.91 1043 accuracy - - 0.84 1810 macro-avg 0.85 0.89 0.87 1810 weighted-avg 0.99 0.84 0.91 1810 ``` --- layout: model title: Recognize Entities OntoNotes - BERT Small author: John Snow Labs name: onto_recognize_entities_bert_small date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [en, open_source, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `small_bert_L4_512` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_small_en_2.7.0_2.4_1607511018075.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_small_en_2.7.0_2.4_1607511018075.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_bert_small') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_small") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu text = ["""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament."""] ner_df = nlu.load('en.ner.onto.bert.small').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |Parliament |ORG | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | |Parliament |ORG | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_small| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: Translate Philippine languages to English Pipeline author: John Snow Labs name: translate_phi_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, phi, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `phi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_phi_en_xx_2.7.0_2.4_1609690604314.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_phi_en_xx_2.7.0_2.4_1609690604314.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_phi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_phi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.phi.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_phi_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from holtin) author: John Snow Labs name: distilbert_qa_holtin_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `holtin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_holtin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771281588.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_holtin_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771281588.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_holtin_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_holtin_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_holtin_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/holtin/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Service agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_service_agreement date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_service_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class service-agreement or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `service-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_service_agreement_en_1.0.0_3.0_1666621014311.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_service_agreement_en_1.0.0_3.0_1666621014311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_service_agreement", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[service-agreement]| |[other]| |[other]| |[service-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_service_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.98 0.96 52 service-agreement 0.96 0.90 0.93 30 accuracy - - 0.95 82 macro avg 0.95 0.94 0.95 82 weighted avg 0.95 0.95 0.95 82 ``` --- layout: model title: Legal Currency Clause Binary Classifier author: John Snow Labs name: legclf_currency_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `currency` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `currency` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_currency_clause_en_1.0.0_3.2_1660123389175.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_currency_clause_en_1.0.0_3.2_1660123389175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_currency_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[currency]| |[other]| |[other]| |[currency]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_currency_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support currency 1.00 0.98 0.99 49 other 0.99 1.00 0.99 96 accuracy - - 0.99 145 macro-avg 0.99 0.99 0.99 145 weighted-avg 0.99 0.99 0.99 145 ``` --- layout: model title: Clinical Deidentification Pipeline (English) author: John Snow Labs name: clinical_deidentification date: 2022-03-03 tags: [deidentification, en, licensed, pipeline, clinical] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true recommended: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_3.4.1_3.0_1646335814560.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_3.4.1_3.0_1646335814560.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification","en","clinical/models") val sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.clinical_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

## Results ```bash Masked with entity labels ------------------------------ Name : , Record date: , # . Dr. , ID, IP . He is a male was admitted to the for cystectomy on . Patient's VIN : , SSN , Driver's license . Phone , , , E-MAIL: . Masked with chars ------------------------------ Name : [**************], Record date: [********], # [****]. Dr. [********], ID[**********], IP [************]. He is a [*********] male was admitted to the [**********] for cystectomy on [******]. Patient's VIN : [***************], SSN [**********], Driver's license [*********]. Phone [************], [***************], [***********], E-MAIL: [*************]. Masked with fixed length chars ------------------------------ Name : ****, Record date: ****, # ****. Dr. ****, ID****, IP ****. He is a **** male was admitted to the **** for cystectomy on ****. Patient's VIN : ****, SSN ****, Driver's license ****. Phone ****, ****, ****, E-MAIL: ****. Obfuscated ------------------------------ Name : Berneta Phenes, Record date: 2093-03-14, # Y5003067. Dr. Dr Gaston Margo, IDOX:8976967, IP 001.001.001.001. He is a 91 male was admitted to the MADONNA REHABILITATION HOSPITAL for cystectomy on 07-22-1994. Patient's VIN : 5eeee44ffff555666, SSN 999-84-3686, Driver's license S99956482. Phone 74 617 042, 1407 west stassney lane, Edmonton, E-MAIL: Carliss@hotmail.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: English asr_asr_with_transformers_wav2vec2 TFWav2Vec2ForCTC from osanseviero author: John Snow Labs name: pipeline_asr_asr_with_transformers_wav2vec2 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_asr_with_transformers_wav2vec2` is a English model originally trained by osanseviero. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_asr_with_transformers_wav2vec2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_asr_with_transformers_wav2vec2_en_4.2.0_3.0_1664043860600.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_asr_with_transformers_wav2vec2_en_4.2.0_3.0_1664043860600.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_asr_with_transformers_wav2vec2', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_asr_with_transformers_wav2vec2", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_asr_with_transformers_wav2vec2| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.6 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_wav2vec2_large_robust_LS960 TFWav2Vec2ForCTC from leonardvorbeck author: John Snow Labs name: pipeline_asr_wav2vec2_large_robust_LS960 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_LS960` is a English model originally trained by leonardvorbeck. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_robust_LS960_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_LS960_en_4.2.0_3.0_1664023749748.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_LS960_en_4.2.0_3.0_1664023749748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_robust_LS960', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_robust_LS960", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_robust_LS960| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.6 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English BertForQuestionAnswering model (from Alexander-Learn) author: John Snow Labs name: bert_qa_Alexander_Learn_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `Alexander-Learn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Alexander_Learn_bert_finetuned_squad_en_4.0.0_3.0_1654535398223.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Alexander_Learn_bert_finetuned_squad_en_4.0.0_3.0_1654535398223.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Alexander_Learn_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Alexander_Learn_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_Alexander-Learn").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Alexander_Learn_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Alexander-Learn/bert-finetuned-squad --- layout: model title: Pipeline to Detect Cellular/Molecular Biology Entities author: John Snow Labs name: ner_cellular_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_cellular](https://nlp.johnsnowlabs.com/2021/03/31/ner_cellular_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_cellular_pipeline_en_4.3.0_3.2_1678863174517.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_cellular_pipeline_en_4.3.0_3.2_1678863174517.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_cellular_pipeline", "en", "clinical/models") text = '''Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_cellular_pipeline", "en", "clinical/models") val text = "Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.cellular.pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------------------------------------------|--------:|------:|:------------|-------------:| | 0 | intracellular signaling proteins | 27 | 58 | protein | 0.763367 | | 1 | human T-cell leukemia virus type 1 promoter | 130 | 172 | DNA | 0.559172 | | 2 | Tax | 186 | 188 | protein | 0.9869 | | 3 | Tax-responsive element 1 | 193 | 216 | DNA | 0.973067 | | 4 | cyclic AMP-responsive members | 237 | 265 | protein | 0.6491 | | 5 | CREB/ATF family | 274 | 288 | protein | 0.85205 | | 6 | transcription factors | 293 | 313 | protein | 0.78645 | | 7 | Tax | 389 | 391 | protein | 0.9584 | | 8 | human T-cell leukemia virus type 1 Tax-responsive element 1 | 396 | 454 | DNA | 0.678156 | | 9 | TRE-1), | 457 | 463 | DNA | 0.7465 | | 10 | lacZ gene | 582 | 590 | DNA | 0.9465 | | 11 | CYC1 promoter | 617 | 629 | DNA | 0.9915 | | 12 | TRE-1 | 663 | 667 | DNA | 0.9989 | | 13 | cyclic AMP response element-binding protein | 695 | 737 | protein | 0.8081 | | 14 | CREB | 740 | 743 | protein | 0.9962 | | 15 | CREB | 749 | 752 | protein | 0.994 | | 16 | GAL4 activation domain | 767 | 788 | protein | 0.802233 | | 17 | GAD | 791 | 793 | protein | 0.9932 | | 18 | reporter gene | 848 | 860 | DNA | 0.78715 | | 19 | Tax | 863 | 865 | protein | 0.9986 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Translate Spanish to English Pipeline author: John Snow Labs name: translate_es_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, es, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `es` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_es_en_xx_2.7.0_2.4_1609685934740.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_es_en_xx_2.7.0_2.4_1609685934740.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_es_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_es_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.es.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_es_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_with_lm TFWav2Vec2ForCTC from aware-ai author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_with_lm` is a German model originally trained by aware-ai. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm_de_4.2.0_3.0_1664098536828.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm_de_4.2.0_3.0_1664098536828.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_german_with_lm| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Greek RobertaForMaskedLM Base Uncased model (from gealexandri) author: John Snow Labs name: roberta_embeddings_palobert_base_greek_uncased_v1 date: 2022-12-12 tags: [el, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: el edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `palobert-base-greek-uncased-v1` is a Greek model originally trained by `gealexandri`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_palobert_base_greek_uncased_v1_el_4.2.4_3.0_1670858886972.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_palobert_base_greek_uncased_v1_el_4.2.4_3.0_1670858886972.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_palobert_base_greek_uncased_v1","el") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_palobert_base_greek_uncased_v1","el") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_palobert_base_greek_uncased_v1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|el| |Size:|314.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/gealexandri/palobert-base-greek-uncased-v1 - https://arxiv.org/abs/1907.11692 - http://www.paloservices.com/ --- layout: model title: Part of Speech for Czech author: John Snow Labs name: pos_ud_pdt date: 2020-05-04 23:32:00 +0800 task: Part of Speech Tagging language: cs edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, cs] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_pdt_cs_2.5.0_2.4_1588622155494.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_pdt_cs_2.5.0_2.4_1588622155494.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_pdt", "cs") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_pdt", "cs") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny."""] pos_df = nlu.load('cs.pos.ud_pdt').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='ADP', metadata={'word': 'Kromě'}), Row(annotatorType='pos', begin=6, end=9, result='DET', metadata={'word': 'toho'}), Row(annotatorType='pos', begin=10, end=10, result='PUNCT', metadata={'word': ','}), Row(annotatorType='pos', begin=12, end=13, result='SCONJ', metadata={'word': 'že'}), Row(annotatorType='pos', begin=15, end=16, result='AUX', metadata={'word': 'je'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_pdt| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|cs| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: SDOH Community Present Binary Classification author: John Snow Labs name: bert_sequence_classifier_sdoh_community_present_status date: 2022-12-18 tags: [en, licensed, clinical, sequence_classification, classifier, community_present, sdoh] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model classifies related to social support such as a family member or friend in the clinical documents. A discharge summary was classified True for Community-Present if the discharge summary had passages related to active social support and False if such passages were not found in the discharge summary. ## Predicted Entities `True`, `False` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_sdoh_community_present_status_en_4.2.2_3.0_1671371389301.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_sdoh_community_present_status_en_4.2.2_3.0_1671371389301.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+-------+ | text| result| +----------------------------------------------------------------------------------------------------+-------+ |Right inguinal hernia repair in childhood Cervical discectomy 3 years ago Umbilical hernia repair...| [True]| |Atrial Septal Defect with Right Atrial Thrombus Pulmonary Hypertension Obesity, Obstructive Sleep...|[False]| +----------------------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_sdoh_community_present_status| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|410.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## Benchmarking ```bash label precision recall f1-score support False 0.95 0.68 0.80 203 True 0.85 0.98 0.91 359 accuracy - - 0.87 562 macro-avg 0.90 0.83 0.85 562 weighted-avg 0.88 0.87 0.87 562 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Tiv author: John Snow Labs name: opus_mt_en_tiv date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tiv, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tiv` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tiv_xx_2.7.0_2.4_1609170083951.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tiv_xx_2.7.0_2.4_1609170083951.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tiv| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Arabic Part of Speech Tagger (from CAMeL-Lab) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_ca_pos_msa date: 2022-05-09 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-ca-pos-msa` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_msa_ar_3.4.2_3.0_1652092479978.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_ca_pos_msa_ar_3.4.2_3.0_1652092479978.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_ca_pos_msa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-ca-pos-msa - https://dl.acm.org/doi/pdf/10.5555/1621804.1621808 - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: German Bert Embeddings author: John Snow Labs name: bert_embeddings_bert_base_german_uncased date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-german-uncased` is a German model orginally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_uncased_de_3.4.2_3.0_1649675856996.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_uncased_de_3.4.2_3.0_1649675856996.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_german_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-german-uncased - https://deepset.ai/german-bert - https://deepset.ai/ - https://spacy.io/ - https://github.com/allenai/scibert - https://github.com/stefan-it/fine-tuned-berts-seq - https://github.com/dbmdz/berts/issues/new --- layout: model title: Persian Named Entity Recognition (from HooshvareLab) author: John Snow Labs name: bert_ner_bert_base_parsbert_armanner_uncased date: 2022-05-09 tags: [bert, ner, token_classification, fa, open_source] task: Named Entity Recognition language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-parsbert-armanner-uncased` is a Persian model orginally trained by `HooshvareLab`. ## Predicted Entities `fac`, `pers`, `pro`, `event`, `org`, `loc` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_parsbert_armanner_uncased_fa_3.4.2_3.0_1652099598361.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_parsbert_armanner_uncased_fa_3.4.2_3.0_1652099598361.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_parsbert_armanner_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|607.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/HooshvareLab/bert-base-parsbert-armanner-uncased - https://arxiv.org/abs/2005.12515 - https://github.com/HaniehP/PersianNER - https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://arxiv.org/abs/2005.12515 - https://tensorflow.org/tfrc - https://hooshvare.com - https://www.linkedin.com/in/m3hrdadfi/ - https://twitter.com/m3hrdadfi - https://github.com/m3hrdadfi - https://www.linkedin.com/in/mohammad-gharachorloo/ - https://twitter.com/MGharachorloo - https://github.com/baarsaam - https://www.linkedin.com/in/marziehphi/ - https://twitter.com/marziehphi - https://github.com/marziehphi - https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/ - https://twitter.com/mmanthouri - https://github.com/mmanthouri - https://hooshvare.com/ - https://www.linkedin.com/company/hooshvare - https://twitter.com/hooshvare - https://github.com/hooshvare - https://www.instagram.com/hooshvare/ - https://www.linkedin.com/in/sara-tabrizi-64548b79/ - https://www.behance.net/saratabrizi - https://www.instagram.com/sara_b_tabrizi/ --- layout: model title: Japanese BertForMaskedLM Base Cased model (from cl-tohoku) author: John Snow Labs name: bert_embeddings_base_japanese_char date: 2022-12-02 tags: [ja, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ja edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-japanese-char` is a Japanese model originally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_char_ja_4.2.4_3.0_1670018147657.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_japanese_char_ja_4.2.4_3.0_1670018147657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_char","ja") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_japanese_char","ja") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_japanese_char| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|334.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-char - https://github.com/google-research/bert - https://github.com/cl-tohoku/bert-japanese/tree/v1.0 - https://github.com/attardi/wikiextractor - https://taku910.github.io/mecab/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: English BertForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: bert_qa_bert_multi_cased_finedtuned_xquad_chaii date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-cased-finedtuned-xquad-chaii` is a English model orginally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finedtuned_xquad_chaii_en_4.0.0_3.0_1654183791284.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finedtuned_xquad_chaii_en_4.0.0_3.0_1654183791284.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_multi_cased_finedtuned_xquad_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_multi_cased_finedtuned_xquad_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xquad_chaii.bert.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_multi_cased_finedtuned_xquad_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/bert-multi-cased-finedtuned-xquad-chaii --- layout: model title: Visual NER on 10K Filings (SEC) author: John Snow Labs name: visualner_10kfilings date: 2023-01-10 tags: [en, licensed] task: OCR Object Detection language: en nav_key: models edition: Visual NLP 4.0.0 spark_version: 3.2 supported: true annotator: VisualDocumentNERv21 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Visual NER team aimed to extract the main key points in the summary page of SEC 10 K filings (Annual reports). ## Predicted Entities `REGISTRANT`, `ADDRESS`, `PHONE`, `DATE`, `EMPLOYERIDNB`, `EXCHANGE`, `STATE`, `STOCKCLASS`, `STOCKVALUE`, `TRADINGSYMBOL`, `FILENUMBER` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/finance-nlp/90.2.Financial_Visual_NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/visualner_10kfilings_en_4.0.0_3.2_1663769328577.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/visualner_10kfilings_en_4.0.0_3.2_1663769328577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage()\ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) img_to_hocr = ImageToHocr()\ .setInputCol("image")\ .setOutputCol("hocr")\ .setIgnoreResolution(False)\ .setOcrParams(["preserve_interword_spaces=0"]) tokenizer = HocrTokenizer()\ .setInputCol("hocr")\ .setOutputCol("token") doc_ner = VisualDocumentNerV21()\ .pretrained("visualner_10kfilings", "en", "clinical/models")\ .setInputCols(["token", "image"])\ .setOutputCol("entities") draw = ImageDrawAnnotations() \ .setInputCol("image") \ .setInputChunksCol("entities") \ .setOutputCol("image_with_annotations") \ .setFontSize(10) \ .setLineWidth(4)\ .setRectColor(Color.red) # OCR pipeline pipeline = PipelineModel(stages=[ binary_to_image, img_to_hocr, tokenizer, doc_ner, draw ]) bin_df = spark.read.format("binaryFile").load('data/t01.jpg') results = pipeline.transform(bin_df).cache() res = results.collect() path_array = f.split(results['path'], '/') results.withColumn('filename', path_array.getItem(f.size(path_array)- 1)) \ .withColumn("exploded_entities", f.explode("entities")) \ .select("filename", "exploded_entities") \ .show(truncate=False) ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val img_to_hocr = new ImageToHocr() .setInputCol("image") .setOutputCol("hocr") .setIgnoreResolution(False) .setOcrParams(Array("preserve_interword_spaces=0")) val tokenizer = new HocrTokenizer() .setInputCol("hocr") .setOutputCol("token") val doc_ner = VisualDocumentNerV21() .pretrained("visualner_10kfilings", "en", "clinical/models") .setInputCols(Array("token", "image")) .setOutputCol("entities") val draw = new ImageDrawAnnotations() .setInputCol("image") .setInputChunksCol("entities") .setOutputCol("image_with_annotations") .setFontSize(10) .setLineWidth(4) .setRectColor(Color.red) # OCR pipeline val pipeline = new PipelineModel().setStages(Array( binary_to_image, img_to_hocr, tokenizer, doc_ner, draw)) val bin_df = spark.read.format("binaryFile").load("data/t01.jpg") val results = pipeline.transform(bin_df).cache() val res = results.collect() val path_array = f.split(results["path"], "/") val results.withColumn("filename", path_array.getItem(f.size(path_array)- 1)) .withColumn(Array("exploded_entities", f.explode("entities"))) .select(Array("filename", "exploded_entities")) .show(truncate=False) ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image10.jpeg) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image10_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} ## Output text ```bash +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ |filename|exploded_entities | +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ |t01.jpg |{named_entity, 712, 716, OTHERS, {confidence -> 96, width -> 74, x -> 1557, y -> 416, word -> Ended, token -> ended, height -> 18}, []} | |t01.jpg |{named_entity, 718, 724, DATE-B, {confidence -> 96, width -> 97, x -> 1639, y -> 416, word -> January, token -> january, height -> 24}, []} | |t01.jpg |{named_entity, 726, 727, DATE-I, {confidence -> 95, width -> 34, x -> 1743, y -> 416, word -> 31,, token -> 31, height -> 22}, []} | |t01.jpg |{named_entity, 730, 733, DATE-I, {confidence -> 96, width -> 54, x -> 1785, y -> 416, word -> 2021, token -> 2021, height -> 18}, []} | |t01.jpg |{named_entity, 735, 744, OTHERS, {confidence -> 91, width -> 143, x -> 1372, y -> 472, word -> Commission, token -> commission, height -> 18}, []} | |t01.jpg |{named_entity, 746, 749, OTHERS, {confidence -> 96, width -> 36, x -> 1523, y -> 472, word -> file, token -> file, height -> 18}, []} | |t01.jpg |{named_entity, 751, 756, OTHERS, {confidence -> 92, width -> 96, x -> 1568, y -> 472, word -> number:, token -> number, height -> 18}, []} | |t01.jpg |{named_entity, 759, 761, FILENUMBER-B, {confidence -> 92, width -> 119, x -> 1675, y -> 472, word -> 001-39495, token -> 001, height -> 18}, []} | |t01.jpg |{named_entity, 769, 773, REGISTRANT-B, {confidence -> 92, width -> 136, x -> 1472, y -> 558, word -> ASANA,, token -> asana, height -> 31}, []} | |t01.jpg |{named_entity, 776, 778, REGISTRANT-I, {confidence -> 95, width -> 72, x -> 1620, y -> 558, word -> INC., token -> inc, height -> 25}, []} +--------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|visualner_10kfilings| |Type:|ocr| |Compatibility:|Visual NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|744.4 MB| ## References SEC 10k filings --- layout: model title: RoBERTa Large CoNLL-03 NER Pipeline author: John Snow Labs name: roberta_large_token_classifier_conll03_pipeline date: 2022-04-23 tags: [open_source, ner, token_classifier, roberta, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [roberta_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/roberta_large_token_classifier_conll03_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_large_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650718129682.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_large_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650718129682.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ruperta_base_finetuned_squadv1","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ruperta_base_finetuned_squadv1","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ruperta_base_finetuned_squadv1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|470.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-squadv1 --- layout: model title: English BertForQuestionAnswering Base Uncased model (from kamalkraj) author: John Snow Labs name: bert_qa_base_uncased_squad_v2.0_finetuned date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squad-v2.0-finetuned` is a English model originally trained by `kamalkraj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_squad_v2.0_finetuned_en_4.0.0_3.0_1657185760559.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_squad_v2.0_finetuned_en_4.0.0_3.0_1657185760559.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_squad_v2.0_finetuned","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_squad_v2.0_finetuned","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_squad_v2.0_finetuned| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kamalkraj/bert-base-uncased-squad-v2.0-finetuned --- layout: model title: Legal Bank accounts Clause Binary Classifier author: John Snow Labs name: legclf_bank_accounts_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `bank-accounts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `bank-accounts` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_bank_accounts_clause_en_1.0.0_3.2_1660122150189.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_bank_accounts_clause_en_1.0.0_3.2_1660122150189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_bank_accounts_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[bank-accounts]| |[other]| |[other]| |[bank-accounts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_bank_accounts_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support bank-accounts 1.00 0.98 0.99 43 other 0.99 1.00 1.00 112 accuracy - - 0.99 155 macro-avg 1.00 0.99 0.99 155 weighted-avg 0.99 0.99 0.99 155 ``` --- layout: model title: Bemba (Zambia) asr_wav2vec2_large_xlsr_bemba TFWav2Vec2ForCTC from csikasote author: John Snow Labs name: asr_wav2vec2_large_xlsr_bemba date: 2022-09-24 tags: [wav2vec2, bem, audio, open_source, asr] task: Automatic Speech Recognition language: bem edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_bemba` is a Bemba (Zambia) model originally trained by csikasote. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_bemba_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_bemba_bem_4.2.0_3.0_1664023099610.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_bemba_bem_4.2.0_3.0_1664023099610.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_bemba", "bem")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_bemba", "bem") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_bemba| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|bem| |Size:|1.2 GB| --- layout: model title: Spanish ElectraForQuestionAnswering model (from hackathon-pln-es) author: John Snow Labs name: electra_qa_biomedtra_small_es_squad2 date: 2022-06-22 tags: [es, open_source, electra, question_answering] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biomedtra-small-es-squad2-es` is a Spanish model originally trained by `hackathon-pln-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_biomedtra_small_es_squad2_es_4.0.0_3.0_1655920306800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_biomedtra_small_es_squad2_es_4.0.0_3.0_1655920306800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_biomedtra_small_es_squad2","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_biomedtra_small_es_squad2","es") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2.electra.small").predict("""¿Cuál es mi nombre?|||"Mi nombre es Clara y vivo en Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_biomedtra_small_es_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|51.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/hackathon-pln-es/biomedtra-small-es-squad2-es --- layout: model title: Detect Chemicals in Medical text (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_chemicals date: 2021-10-19 tags: [berfortokenclassification, ner, chemicals, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract different types of chemical compounds mentioned in text using pretrained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `CHEM` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_BERT_TOKEN_CLASSIFIER/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_en_3.3.0_2.4_1634649785035.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_en_3.3.0_2.4_1634649785035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_chemicals", "en", "clinical/models")\ .setInputCols("token", "document")\ .setOutputCol("ner")\ .setCaseSensitive(True) ner_converter = NerConverter()\ .setInputCols(["document","token","ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier, ner_converter]) p_model = pipeline.fit(spark.createDataFrame(pd.DataFrame({'text': ['']}))) test_sentence = """The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""" result = p_model.transform(spark.createDataFrame(pd.DataFrame({'text': [test_sentence]}))) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_chemicals", "en", "clinical/models") .setInputCols(Array("token", "document")) .setOutputCol("ner") .setCaseSensitive(True) val ner_converter = new NerConverter() .setInputCols(Array("document","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier, ner_converter)) val data = Seq("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_chemical").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. "A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |p - choloroaniline |CHEM | |chlorhexidine - digluconate|CHEM | |kanamycin |CHEM | |colistin |CHEM | |povidone - iodine |CHEM | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemicals| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|512| ## Data Source This model is trained on a custom dataset by John Snow Labs. ## Benchmarking ```bash label precision recall f1-score support B-CHEM 0.99 0.92 0.95 30731 I-CHEM 0.99 0.93 0.96 31270 accuracy - - 0.93 62001 macro-avg 0.96 0.95 0.96 62001 weighted-avg 0.99 0.93 0.96 62001 ``` --- layout: model title: Stop Words Cleaner for Japanese author: John Snow Labs name: stopwords_ja date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: ja edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, ja] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_ja_ja_2.5.4_2.4_1594742438927.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_ja_ja_2.5.4_2.4_1594742438927.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_ja", "ja") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("北の王であることを除いて、ジョン・スノーはイギリスの医師であり、麻酔と医療衛生の開発のリーダーです。") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_ja", "ja") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("北の王であることを除いて、ジョン・スノーはイギリスの医師であり、麻酔と医療衛生の開発のリーダーです。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""北の王であることを除いて、ジョン・スノーはイギリスの医師であり、麻酔と医療衛生の開発のリーダーです。"""] stopword_df = nlu.load('ja.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=49, result='北の王であることを除いて、ジョン・スノーはイギリスの医師であり、麻酔と医療衛生の開発のリーダーです。', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_ja| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ja| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Legal Headings Clause Binary Classifier author: John Snow Labs name: legclf_headings_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `headings` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `headings` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_headings_clause_en_1.0.0_3.2_1660123578960.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_headings_clause_en_1.0.0_3.2_1660123578960.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_headings_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[headings]| |[other]| |[other]| |[headings]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_headings_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support headings 0.95 0.97 0.96 131 other 0.98 0.98 0.98 252 accuracy - - 0.97 383 macro-avg 0.97 0.97 0.97 383 weighted-avg 0.97 0.97 0.97 383 ``` --- layout: model title: Fast Neural Machine Translation Model from Afrikaans to Spanish author: John Snow Labs name: opus_mt_af_es date: 2021-06-01 tags: [open_source, seq2seq, translation, af, es, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: af target languages: es {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_af_es_xx_3.1.0_2.4_1622555083309.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_af_es_xx_3.1.0_2.4_1622555083309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_af_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Living Species(w2v_cc_300d) author: John Snow Labs name: ner_living_species date: 2022-06-22 tags: [pt, ner, clinical, licensed] task: Named Entity Recognition language: pt edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Portuguese which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `w2v_cc_300d` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pt_3.5.3_3.0_1655922628486.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pt_3.5.3_3.0_1655922628486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","pt")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species", "pt","clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","pt") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species", "pt","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("pt.med_ner.living_species").predict("""Uma rapariga de 16 anos com um historial pessoal de asma apresentou ao departamento de dermatologia com lesões cutâneas assintomáticas que tinham estado presentes durante 2 meses. A paciente tinha sido tratada com creme corticosteróide devido a uma suspeita inicial de eczema atópico, apesar do qual apresentava um crescimento progressivo marcado das lesões. Tinha um gato doméstico que ela nunca tinha levado ao veterinário. O exame físico revelou placas em forma de anel com uma borda periférica activa na parte superior das costas e nos aspectos laterais do pescoço e da face. Cultura local obtida por raspagem de tapete isolado Trichophyton rubrum. Com base em dados clínicos e cultura, foi estabelecido o diagnóstico de tinea incognito.""") ```

## Results ```bash +-------------------+-------+ |ner_chunk |label | +-------------------+-------+ |rapariga |HUMAN | |pessoal |HUMAN | |paciente |HUMAN | |gato |SPECIES| |veterinário |HUMAN | |Trichophyton rubrum|SPECIES| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|pt| |Size:|15.1 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.80 0.94 0.87 2832 B-SPECIES 0.52 0.89 0.65 2810 I-HUMAN 0.80 0.62 0.70 180 I-SPECIES 0.68 0.82 0.74 1107 micro-avg 0.64 0.89 0.75 6929 macro-avg 0.70 0.82 0.74 6929 weighted-avg 0.67 0.89 0.76 6929 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from fadhilarkan) author: John Snow Labs name: distilbert_qa_fadhilarkan_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `fadhilarkan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_fadhilarkan_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725246680.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_fadhilarkan_base_uncased_finetuned_squad_en_4.0.0_3.0_1654725246680.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_fadhilarkan_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_fadhilarkan_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_fadhilarkan").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_fadhilarkan_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/fadhilarkan/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-26 tags: [bg, licensed, ner, legal, mapa] task: Named Entity Recognition language: bg edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Bulgarian` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_bg_1.0.0_3.0_1682548782666.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_bg_1.0.0_3.0_1682548782666.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_base_bg_cased", "bg")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "bg", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""7 В окончателно решение № 1072 на Curtea de Apel București ( Апелативен съд Букурещ, Румъния ), 3-то гражданско отделение за малолетни и непълнолетни лица и семейноправни въпроси, от 12 юни 2013г., което е приложено към акта за преюдициално запитване и представено от г‑н Liberato, се уточнява, че„ [с] ъдът приема, че страните са сключили брак в Италия през октомври 2005 г. и до октомври 2006 г. са живели ту в Румъния, ту в Италия."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +----------------+---------+ |chunk |ner_label| +----------------+---------+ |Букурещ, Румъния|ADDRESS | |12 юни 2013г., |DATE | |г‑н Liberato |PERSON | |Италия |ADDRESS | |октомври 2005 г.|DATE | |октомври 2006 г.|DATE | |Румъния |ADDRESS | |Италия |ADDRESS | +----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|bg| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.86 0.75 0.80 8 AMOUNT 1.00 0.64 0.78 11 DATE 0.97 0.97 0.97 65 ORGANISATION 0.81 0.86 0.83 35 PERSON 0.87 0.84 0.85 56 macro-avg 0.90 0.87 0.89 175 macro-avg 0.90 0.81 0.85 175 weighted-avg 0.90 0.87 0.89 175 ``` --- layout: model title: English BertForQuestionAnswering model (from JAlexis) author: John Snow Labs name: bert_qa_Bertv1_fine date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Bertv1_fine` is a English model orginally trained by `JAlexis`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Bertv1_fine_en_4.0.0_3.0_1654176446585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Bertv1_fine_en_4.0.0_3.0_1654176446585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Bertv1_fine","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Bertv1_fine","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.cord19.bert.by_JAlexis").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Bertv1_fine| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/JAlexis/Bertv1_fine --- layout: model title: Word2Vec Embeddings in Turkish (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, tr, open_source] task: Embeddings language: tr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tr_3.4.1_3.0_1647463837323.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tr_3.4.1_3.0_1647463837323.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Hasanmurad_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Hasanmurad/dummy-model --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-28 tags: [sk, licensed, ner, legal, mapa] task: Named Entity Recognition language: sk edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Slovak` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_sk_1.0.0_3.0_1682674803309.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_sk_1.0.0_3.0_1682674803309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_base_slovak_legal","sk")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "sk", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Návrhom podaným 22. mája 2007 na Tribunale di Teramo ( súd v Terame, Taliansko ) požiadal pán Liberato o rozluku a o zverenie syna do svojej starostlivosti."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-------------+---------+ |chunk |ner_label| +-------------+---------+ |22. mája 2007|DATE | |Terame |ADDRESS | |Taliansko |ADDRESS | |pán Liberato |PERSON | +-------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|sk| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.88 0.85 0.86 26 AMOUNT 1.00 1.00 1.00 4 DATE 0.92 0.88 0.90 50 ORGANISATION 0.79 0.61 0.69 31 PERSON 0.66 0.86 0.75 44 micro-avg 0.80 0.82 0.81 155 macro-avg 0.85 0.84 0.84 155 weighted-avg 0.81 0.82 0.81 155 ``` --- layout: model title: Notice Clause Relation Extraction Model author: John Snow Labs name: legre_notice_clause_xs date: 2022-12-17 tags: [en, legal, relations, redl, licensed, tensorflow] task: Relation Extraction language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Relation Extraction model aimed to be used in notice clauses, to retrieve relations between entities as NOTICE_PARTY, ADDRESS, EMAIL, TITLE etc. Make sure you run this model only on the NER entities in notice clauses, after you filter them using `legclf_notice_clause` ## Predicted Entities `has_notice_party`, `has_address`, `has_person`, `has_phone`, `has_fax`, `has_title`, `has_email`, `has_department` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legre_notice_clause_xs_en_1.0.0_3.0_1671280929569.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legre_notice_clause_xs_en_1.0.0_3.0_1671280929569.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = nlp.SentenceDetectorDLModel.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence")\ .setCustomBounds(["\n\n"])\ .setUseCustomBoundsOnly(True) tokenizer = nlp.Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos_tagger = nlp.PerceptronModel.pretrained()\ .setInputCols(["sentence", "token"])\ .setOutputCol("pos_tags") dependency_parser = nlp.DependencyParserModel() \ .pretrained("dependency_conllu", "en") \ .setInputCols(["sentence", "pos_tags", "token"]) \ .setOutputCol("dependencies") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = legal.NerModel.pretrained('legner_notice_clause', 'en', 'legal/models') \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = nlp.NerConverter() \ .setInputCols(["sentence","token","ner"]) \ .setOutputCol("ner_chunk") re_filter = legal.RENerChunksFilter()\ .setInputCols(["ner_chunk", "dependencies"])\ .setOutputCol("re_ner_chunks")\ .setMaxSyntacticDistance(12)\ .setRelationPairs(['NAME-NOTICE_PARTY','NAME-ADDRESS','NAME-PERSON', 'NAME-TITLE','NAME-EMAIL','NAME-PHONE', 'NAME-FAX', 'NAME-DEPARTMENT']) reDL = legal.RelationExtractionDLModel.pretrained("legre_notice_clause_xs", "en", "legal/models") \ .setPredictionThreshold(0.1) \ .setInputCols(["re_ner_chunks", "sentence"]) \ .setOutputCol("relations") pipeline = nlp.Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos_tagger, dependency_parser, embeddings, ner_model, ner_converter, re_filter, reDL]) empty_df = spark.createDataFrame([['']]).toDF("text") re_model = pipeline.fit(empty_df) light_model = nlp.LightPipeline(re_model) text = """The addresses for notices shall be: IBM MSL 8501 IBM Drive 200 Baker Avenue Charlotte, NC 28262 Concord, MA 01742 Attn: MSL Project Office Attn: General Counsel Telephone: 704-594-1964 Telephone: 978-287-5630 Facsimile: 704-594-4108 Facsimile: 978-287-5635 Either Party may change its address for this section by giving written notice to the other Party.""" result = light_model.fullAnnotate(text) ```

## Results ```bash | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---------------------|------------|------------------|----------------|------------|---------------|------------------|----------------|------------------------------------------------------|---------------| | has_address | NAME | 36 | 42 | IBM MSL | ADDRESS | 44 | 112 | 8501 IBM Drive 200 Baker Avenue Charlotte, NC ... | 0.9997987 | | has_notice_party | NAME | 36 | 42 | IBM MSL | DEPARTMENT | 120 | 137 | MSL Project Office | 0.34552842 | | has_title | NAME | 36 | 42 | IBM MSL | TITLE | 145 | 159 | General Counsel | 0.48349348 | | has_phone | NAME | 36 | 42 | IBM MSL | PHONE | 173 | 184 | 704-594-1964 | 0.99517375 | | has_phone | NAME | 36 | 42 | IBM MSL | PHONE | 197 | 208 | 978-287-5630 | 0.9961247 | | has_fax | NAME | 36 | 42 | IBM MSL | FAX | 221 | 232 | 704-594-4108 | 0.99340916 | | has_fax | NAME | 36 | 42 | IBM MSL | FAX | 245 | 256 | 978-287-5635 | 0.97187006 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legre_notice_clause_xs| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|402.6 MB| ## References In-house dataset ## Benchmarking ```bash label Recall Precision F1 Support has_address 0.976 1.000 0.988 41 has_department 0.667 1.000 0.800 3 has_email 1.000 1.000 1.000 7 has_fax_phone 1.000 1.000 1.000 8 has_notice_party 1.000 0.955 0.977 42 has_person 1.000 0.938 0.968 15 has_title 0.875 0.933 0.903 16 other 1.000 1.000 1.000 68 Avg. 0.940 0.978 0.954 - Weighted-Avg. 0.980 0.980 0.979 - ``` --- layout: model title: English BertForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_bert_triplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657190994171.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1657190994171.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_rule_based_triplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_bert_triplet_epochs_1_shard_1_squad2.0 --- layout: model title: Chinese BertForMaskedLM Small Cased model (from hfl) author: John Snow Labs name: bert_embeddings_chinese_lert_small date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-lert-small` is a Chinese model originally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_lert_small_zh_4.2.4_3.0_1670021149485.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_lert_small_zh_4.2.4_3.0_1670021149485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_lert_small","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_lert_small","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_lert_small| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|57.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-lert-small - https://github.com/ymcui/LERT/blob/main/README_EN.md - https://arxiv.org/abs/2211.05344 --- layout: model title: Legal Administration Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_administration_agreement date: 2022-11-24 tags: [en, legal, classification, agreement, administration, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_administration_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `administration-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `administration-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_administration_agreement_en_1.0.0_3.0_1669290751753.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_administration_agreement_en_1.0.0_3.0_1669290751753.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[administration-agreement]| |[other]| |[other]| |[administration-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_administration_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support administration-agreement 0.93 0.93 0.93 29 other 0.98 0.98 0.98 90 accuracy - - 0.97 119 macro-avg 0.95 0.95 0.95 119 weighted-avg 0.97 0.97 0.97 119 ``` --- layout: model title: Legal Litigation Clause Binary Classifier author: John Snow Labs name: legclf_litigation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `litigation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `litigation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_litigation_clause_en_1.0.0_3.2_1660122631845.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_litigation_clause_en_1.0.0_3.2_1660122631845.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_litigation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[litigation]| |[other]| |[other]| |[litigation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_litigation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support litigation 1.00 0.89 0.94 27 other 0.97 1.00 0.99 103 accuracy - - 0.98 130 macro-avg 0.99 0.94 0.96 130 weighted-avg 0.98 0.98 0.98 130 ``` --- layout: model title: Greek BertForQuestionAnswering model (from Danastos) author: John Snow Labs name: bert_qa_qacombination_bert_el_Danastos date: 2022-06-03 tags: [el, open_source, question_answering, bert] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qacombination_bert_el` is a Greek model orginally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_qacombination_bert_el_Danastos_el_4.0.0_3.0_1654249989643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_qacombination_bert_el_Danastos_el_4.0.0_3.0_1654249989643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_qacombination_bert_el_Danastos","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_qacombination_bert_el_Danastos","el") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("el.answer_question.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_qacombination_bert_el_Danastos| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|el| |Size:|421.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/qacombination_bert_el --- layout: model title: Extract Granular Anatomical Entities from Oncology Texts author: John Snow Labs name: ner_oncology_anatomy_granular date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner, anatomy] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extractions mentions of anatomical entities using granular labels. Definitions of Predicted Entities: - `Direction`: Directional and laterality terms, such as "left", "right", "bilateral", "upper" and "lower". - `Site_Bone`: Anatomical terms that refer to the human skeleton. - `Site_Brain`: Anatomical terms that refer to the central nervous system (including the brain stem and the cerebellum). - `Site_Breast`: Anatomical terms that refer to the breasts. - `Site_Liver`: Anatomical terms that refer to the liver. - `Site_Lung`: Anatomical terms that refer to the lungs. - `Site_Lymph_Node`: Anatomical terms that refer to lymph nodes, excluding adenopathies. - `Site_Other_Body_Part`: Relevant anatomical terms that are not included in the rest of the anatomical entities. ## Predicted Entities `Direction`, `Site_Bone`, `Site_Brain`, `Site_Breast`, `Site_Liver`, `Site_Lung`, `Site_Lymph_Node`, `Site_Other_Body_Part` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_granular_en_4.0.0_3.0_1666722590194.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_anatomy_granular_en_4.0.0_3.0_1666722590194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:--------|:------------| | left | Direction | | breast | Site_Breast | | lungs | Site_Lung | | liver | Site_Liver | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_anatomy_granular| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Direction 822 221 162 984 0.79 0.84 0.81 Site_Lymph_Node 481 38 70 551 0.93 0.87 0.90 Site_Breast 88 14 59 147 0.86 0.60 0.71 Site_Other_Body_Part 604 184 897 1501 0.77 0.40 0.53 Site_Bone 252 74 61 313 0.77 0.81 0.79 Site_Liver 178 92 56 234 0.66 0.76 0.71 Site_Lung 398 98 161 559 0.80 0.71 0.75 Site_Brain 197 44 82 279 0.82 0.71 0.76 macro_avg 3020 765 1548 4568 0.80 0.71 0.74 micro_avg 3020 765 1548 4568 0.80 0.66 0.71 ``` --- layout: model title: English RobertaForQuestionAnswering (from comacrae) author: John Snow Labs name: roberta_qa_roberta_edav3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-edav3` is a English model originally trained by `comacrae`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_edav3_en_4.0.0_3.0_1655735808428.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_edav3_en_4.0.0_3.0_1655735808428.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_edav3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_edav3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.edav3.by_comacrae").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_edav3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/comacrae/roberta-edav3 --- layout: model title: Pipeline to Detect PHI for Deidentification (Subentity- Augmented) author: John Snow Labs name: ner_deid_subentity_augmented_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_subentity_augmented](https://nlp.johnsnowlabs.com/2021/09/03/ner_deid_subentity_augmented_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_pipeline_en_3.4.1_3.0_1647870542439.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_subentity_augmented_pipeline_en_3.4.1_3.0_1647870542439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_deid_subentity_augmented_pipeline", "en", "clinical/models") pipeline.annotate("A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.") ``` ```scala val pipeline = new PretrainedPipeline("ner_deid_subentity_augmented_pipeline", "en", "clinical/models") pipeline.annotate("A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.subentity_ner_augmented.pipeline").predict("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25-year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227. Patient's complaints first surfaced when he started working for Brothers Coal-Mine.""") ```

## Results ```bash +-----------------------------+-------------+ |chunk |ner_label | +-----------------------------+-------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson, Ora |PATIENT | |7194334 |MEDICALRECORD| |01/13/93 |DATE | |Oliveira |DOCTOR | |25-year-old |AGE | |1-11-2000 |DATE | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street. |STREET | |(302) 786-5227 |PHONE | |Brothers Coal-Mine |ORGANIZATION | +-----------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_subentity_augmented_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_doddle124578 TFWav2Vec2ForCTC from doddle124578 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_doddle124578 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_doddle124578` is a English model originally trained by doddle124578. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_doddle124578_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_doddle124578_en_4.2.0_3.0_1664036087586.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_doddle124578_en_4.2.0_3.0_1664036087586.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_doddle124578", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_doddle124578", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_doddle124578| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_final_784824209 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-final-784824209` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824209_en_4.3.1_3.0_1678134230054.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824209_en_4.3.1_3.0_1678134230054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824209","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824209","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_final_784824209| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-final-784824209 --- layout: model title: Fast Neural Machine Translation Model from English to Kaonde author: John Snow Labs name: opus_mt_en_kqn date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, kqn, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `kqn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_kqn_xx_2.7.0_2.4_1609168468183.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_kqn_xx_2.7.0_2.4_1609168468183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_kqn| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Relationship of the parties Clause Binary Classifier author: John Snow Labs name: legclf_relationship_of_the_parties_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `relationship_of_the_parties` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `relationship_of_the_parties` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_relationship_of_the_parties_clause_en_1.0.0_3.2_1660122908778.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_relationship_of_the_parties_clause_en_1.0.0_3.2_1660122908778.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_relationship_of_the_parties_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[relationship_of_the_parties]| |[other]| |[other]| |[relationship_of_the_parties]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_relationship_of_the_parties_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 1.00 0.99 241 relationship of the parties 0.99 0.96 0.97 78 accuracy - - 0.99 319 macro-avg 0.99 0.98 0.98 319 weighted-avg 0.99 0.99 0.99 319 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_ff9000 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-ff9000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff9000_en_4.3.0_3.0_1675121175904.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_ff9000_en_4.3.0_3.0_1675121175904.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_ff9000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_ff9000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_ff9000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|316.3 MB| ## References - https://huggingface.co/google/t5-efficient-small-ff9000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English RobertaForQuestionAnswering Cased model (from pierrerappolt) author: John Snow Labs name: roberta_qa_cart date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cart` is a English model originally trained by `pierrerappolt`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cart_en_4.2.4_3.0_1669985087510.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cart_en_4.2.4_3.0_1669985087510.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cart","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cart","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cart| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/pierrerappolt/cart --- layout: model title: Fast Neural Machine Translation Model from Italian to English author: John Snow Labs name: opus_mt_it_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, it, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `it` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_it_en_xx_2.7.0_2.4_1609163790941.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_it_en_xx_2.7.0_2.4_1609163790941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_it_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Mapping Entities (Major Clinical Concepts) with Corresponding UMLS CUI Codes author: John Snow Labs name: umls_major_concepts_mapper date: 2022-07-11 tags: [umls, chunk_mapper, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps entities (Major Clinical Concepts) with corresponding UMLS CUI codes. ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_major_concepts_mapper_en_4.0.0_3.0_1657579020337.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_major_concepts_mapper_en_4.0.0_3.0_1657579020337.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+---------+ |ner_chunk |umls_code| +-------------------------+---------+ |pustules |C0241157 | |stairs |C4300351 | |quadriceps tendon rupture|C0263968 | +-------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_major_concepts_mapper| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|37.0 MB| ## References Data sampled from https://www.nlm.nih.gov/research/umls/index.html --- layout: model title: T5 for grammar error correction author: John Snow Labs name: t5_grammar_error_corrector date: 2022-01-12 tags: [t5, en, open_source] task: Text Generation language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a text-to-text model fine tuned to correct grammatical errors when the task is set to "gec:". It is based on Prithiviraj Damodaran's Gramformer model. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5_LINGUISTIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_grammar_error_corrector_en_3.4.0_3.0_1641983182673.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_grammar_error_corrector_en_3.4.0_3.0_1641983182673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from sparknlp.base import * from sparknlp.annotator import * spark = sparknlp.start() documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = T5Transformer.pretrained("t5_grammar_error_corrector") \ .setTask("gec:") \ .setInputCols(["documents"]) \ .setMaxOutputLength(200) \ .setOutputCol("corrections") pipeline = Pipeline().setStages([documentAssembler, t5]) data = spark.createDataFrame([["He are moving here."]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("corrections.result").show(truncate=False) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.seq2seq.T5Transformer import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer.pretrained("t5_grammar_error_corrector") .setTask("gec:") .setMaxOutputLength(200) .setInputCols("documents") .setOutputCol("corrections") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("He are moving here.").toDF("text") val result = pipeline.fit(data).transform(data) result.select("corrections.result").show(false) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5.grammar_error_corrector").predict("""He are moving here.""") ```

## Results ```bash +--------------------+ |result | +--------------------+ |[He is moving here.]| +--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_grammar_error_corrector| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[corrections]| |Language:|en| |Size:|926.2 MB| ## Data Source Model originally from the transformers library: https://huggingface.co/prithivida/grammar_error_correcter_v1 --- layout: model title: Legal Distribution Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_distribution_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, distribution, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_distribution_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `distribution-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `distribution-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_distribution_agreement_bert_en_1.0.0_3.0_1669310752189.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_distribution_agreement_bert_en_1.0.0_3.0_1669310752189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_distribution_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[distribution-agreement]| |[other]| |[other]| |[distribution-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_distribution_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support distribution-agreement 0.92 0.95 0.94 38 other 0.97 0.95 0.96 65 accuracy - - 0.95 103 macro-avg 0.95 0.95 0.95 103 weighted-avg 0.95 0.95 0.95 103 ``` --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_becasincentivos4 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-becasIncentivos4` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos4_es_4.3.0_3.0_1674218146985.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_becasincentivos4_es_4.3.0_3.0_1674218146985.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos4","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_becasincentivos4","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_becasincentivos4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|459.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-becasIncentivos4 --- layout: model title: Tamil Lemmatizer author: John Snow Labs name: lemma date: 2021-04-02 tags: [ta, open_source, lemmatizer] task: Lemmatization language: ta edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: LemmatizerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a dictionary-based lemmatizer that assigns all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_ta_3.0.0_3.0_1617388293492.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_ta_3.0.0_3.0_1617388293492.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------+ | lemma| +-------------+ | கட்சி| | வெற்றி| | பெற்றதோடு| | பெற்றத்| | ஓடு| | ஆட்சி| | உள்| | கட்சிக்கு| | ஒரு| | மாற்றுக்| | கட்சியாக| |வளர்ந்துள்ளது| | வளர்ந்த்| | உள்| | .| +-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|ta| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) version 2.7. ## Benchmarking ```bash Precision=0.62, Recall=0.58, F1-score=0.6 ``` --- layout: model title: Multilingual XLMRoBerta Embeddings (from hfl) author: John Snow Labs name: xlmroberta_embeddings_cino_large_v2 date: 2022-05-13 tags: [zh, ko, open_source, xlm_roberta, embeddings, xx, cino] task: Embeddings language: xx edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `cino-large-v2` is a Multilingual model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_large_v2_xx_3.4.4_3.0_1652439564348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_large_v2_xx_3.4.4_3.0_1652439564348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_cino_large_v2| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|1.7 GB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/cino-large-v2 - https://github.com/ymcui/Chinese-Minority-PLM - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology --- layout: model title: Chinese Word Segmentation author: John Snow Labs name: wordseg_ctb9 date: 2021-01-03 task: Word Segmentation language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [zh] supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WordSegmenterModel (WSM) is based on maximum entropy probability model to detect word boundaries in Chinese text. Chinese text is written without white space between the words, and a computer-based application cannot know _a priori_ which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. We trained this model on the the [Chinese Penn Treebank](https://www.cs.brandeis.edu/~clp/ctb/) version 9 data set following the research paper by Nianwen Xue. References: - Xue, Nianwen, et al. Chinese Treebank 9.0 LDC2016T13. Web Download. Philadelphia: Linguistic Data Consortium, 2016. - Xue, Nianwen. "Chinese word segmentation as character tagging." International Journal of Computational Linguistics & Chinese Language Processing, Volume 8, Number 1, February 2003: Special Issue on Word Formation and Chinese Language Processing. 2003.). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_ctb9_zh_2.7.0_2.4_1609691676660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_ctb9_zh_2.7.0_2.4_1609691676660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline as a substitute of the Tokenizer stage.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") word_segmenter = WordSegmenterModel.pretrained("wordseg_ctb9")\ .setInputCols("document")\ .setOutputCol("token") pipeline = Pipeline(stages=[ document_assembler, word_segmenter ]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['然而，这样的处理也衍生了一些问题。']], ["text"]) result = ws_model.transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val word_segmenter = WordSegmenterModel.pretrained("wordseg_ctb9", "zh") .setInputCols("document") .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("然而，这样的处理也衍生了一些问题。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，这样的处理也衍生了一些问题。"""] token_df = nlu.load('zh.segment_words.ctb9').predict(text, output_level='token') token_df ```

## Results ```bash +----------------------------------+--------------------------------------------------------+ |text |result | +----------------------------------+--------------------------------------------------------+ |然而，这样的处理也衍生了一些问题。|[然而, ，, 这样, 的, 处理, 也, 衍生, 了, 一些, 问题, 。]| +----------------------------------+--------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_ctb9| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|zh| ## Data Source ctb9_train.chartag ## Benchmarking ```bash | Model | precision | recall | f1-score | |---------------|--------------|--------------|--------------| | WORSEG_CTB | 0,6453 | 0,6341 | 0,6397 | | WORDSEG_WEIBO | 0,5454 | 0,5655 | 0,5553 | | WORDSEG_MSR | 0,5984 | 0,6088 | 0,6035 | | WORDSEG_PKU | 0,6094 | 0,6321 | 0,6206 | | WORDSEG_LARGE | 0,6326 | 0,6269 | 0,6297 | ``` --- layout: model title: English image_classifier_vit_rust_image_classification_9 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_9 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_9` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_9_en_4.1.0_3.0_1660171188424.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_9_en_4.1.0_3.0_1660171188424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_9", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_9", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_9| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Named Entity Recognition - BERT Medium (OntoNotes) author: John Snow Labs name: onto_small_bert_L8_512 date: 2020-12-05 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ner, open_source, en] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Onto is a Named Entity Recognition (or NER) model trained on OntoNotes 5.0. It can extract up to 18 entities such as people, places, organizations, money, time, date, etc. This model uses the pretrained `small_bert_L8_512` embeddings model from the `BertEmbeddings` annotator as an input. ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_small_bert_L8_512_en_2.7.0_2.4_1607199531477.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_small_bert_L8_512_en_2.7.0_2.4_1607199531477.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... ner_onto = NerDLModel.pretrained("onto_small_bert_L8_512", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val ner_onto = NerDLModel.pretrained("onto_small_bert_L8_512", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.onto.bert.small_l8_512').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PERSON | |October 28, 1955 |DATE | |American |NORP | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PERSON | |May 2014 |DATE | |the 1970s and 1980s |DATE | |Seattle |GPE | |Washington |GPE | |Gates |PERSON | |Paul Allen |PERSON | |1975 |DATE | |Albuquerque |GPE | |New Mexico |GPE | |Gates |PERSON | |January 2000 |DATE | |the late 1990s |DATE | |Gates |PERSON | |June 2006 |DATE | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_small_bert_L8_512| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source The model is trained based on data from [OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) ## Benchmarking ```bash Micro-average: prec: 0.8849518, rec: 0.85147995, f1: 0.8678933 CoNLL Eval: processed 152728 tokens with 11257 phrases; found: 11073 phrases; correct: 9556. accuracy: 97.26%; 9556 11257 11073 precision: 86.30%; recall: 84.89%; FB1: 85.59 CARDINAL: 798 935 929 precision: 85.90%; recall: 85.35%; FB1: 85.62 929 DATE: 1410 1602 1654 precision: 85.25%; recall: 88.01%; FB1: 86.61 1654 EVENT: 23 63 44 precision: 52.27%; recall: 36.51%; FB1: 42.99 44 FAC: 79 135 121 precision: 65.29%; recall: 58.52%; FB1: 61.72 121 GPE: 2097 2240 2244 precision: 93.45%; recall: 93.62%; FB1: 93.53 2244 LANGUAGE: 9 22 11 precision: 81.82%; recall: 40.91%; FB1: 54.55 11 LAW: 14 40 20 precision: 70.00%; recall: 35.00%; FB1: 46.67 20 LOC: 111 179 152 precision: 73.03%; recall: 62.01%; FB1: 67.07 152 MONEY: 282 314 320 precision: 88.12%; recall: 89.81%; FB1: 88.96 320 NORP: 755 841 889 precision: 84.93%; recall: 89.77%; FB1: 87.28 889 ORDINAL: 169 195 201 precision: 84.08%; recall: 86.67%; FB1: 85.35 201 ORG: 1368 1795 1624 precision: 84.24%; recall: 76.21%; FB1: 80.02 1624 PERCENT: 309 349 351 precision: 88.03%; recall: 88.54%; FB1: 88.29 351 PERSON: 1816 1988 2037 precision: 89.15%; recall: 91.35%; FB1: 90.24 2037 PRODUCT: 42 76 67 precision: 62.69%; recall: 55.26%; FB1: 58.74 67 QUANTITY: 85 105 108 precision: 78.70%; recall: 80.95%; FB1: 79.81 108 TIME: 137 212 222 precision: 61.71%; recall: 64.62%; FB1: 63.13 222 WORK_OF_ART: 52 166 79 precision: 65.82%; recall: 31.33%; FB1: 42.45 79 ``` --- layout: model title: Emotion Detection Classifier author: John Snow Labs name: Emotion Classifier class: ClassifierDLModel language: en nav_key: models repository: public/models date: 03/07/2020 task: Text Classification edition: Spark NLP 2.5.3 spark_version: 2.4 tags: [open_source, en, classifier, emotion] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Automatically identify Joy, Surprise, Fear, Sadness in Tweets using out pretrained Spark NLP DL classifier. {:.h2_title} ## Predicted Entities ``surprise``, ``sadness``, ``fear``, ``joy``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_EN_EMOTION/){:.button.button-orange}
[Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN_EMOTION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}
[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_emotion_en_2.5.3_2.4_1593783319297.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_emotion_en_2.5.3_2.4_1593783319297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3}
## How to use

{:.h2_title} ## Results ```bash +-------------------------------------------------------------------------------------------------------------------------------------+---------+ |document |sentiment| +-------------------------------------------------------------------------------------------------------------------------------------+---------+ |@Mira I just saw you on live t.v!! |surprise | |Just home from group celebration - dinner at Trattoria Gianni, then Hershey Felder's performance - AMAZING!! |joy | |Nooooo! My dad turned off the internet so I can't listen to band music! |sadness | |My soul has just been pierced by the most evil look from @rickosborneorg. A mini panic attack and chill in bones followed soon after.|fear | +-------------------------------------------------------------------------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| | Model Name | classifierdl_use_emotion | | Model Class | ClassifierDLModel | | Spark Compatibility | 2.5.3 | | Spark NLP Compatibility | 2.4 | | License | open source | | Edition | public | | Input Labels | [document, sentence_embeddings]| | Output Labels | [class] | | Language | en | | Upstream Dependencies | tfhub_use | {:.h2_title} ## Data Source This model is trained on multiple datasets inlcuding youtube comments, twitter and ISEAR dataset. --- layout: model title: English RobertaForQuestionAnswering Large Cased model (from akdeniz27) author: John Snow Labs name: roberta_qa_large_cuad date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-cuad` is a English model originally trained by `akdeniz27`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_large_cuad_en_4.2.4_3.0_1669987585871.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_large_cuad_en_4.2.4_3.0_1669987585871.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_cuad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_large_cuad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_large_cuad| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/akdeniz27/roberta-large-cuad - https://www.atticusprojectai.org/cuad - https://github.com/TheAtticusProject/cuad - https://arxiv.org/abs/2103.06268 - https://aclanthology.org/2021.acl-long.330.pdf - https://dl.acm.org/doi/pdf/10.1145/3442188.3445922 - https://drive.google.com/file/d/1of37X0hAhECQ3BN_004D8gm6V88tgZaB/view?usp=sharing - https://zenodo.org/record/4599830 - https://mlco2.github.io/impact#compute - https://arxiv.org/abs/1910.09700 - https://www.atticusprojectai.org/cuad - https://www.atticusprojectai.org/ --- layout: model title: Legal Termination Clause Binary Classifier author: John Snow Labs name: legclf_termination_clause date: 2023-02-13 tags: [en, legal, classification, termination, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `termination` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `termination`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_termination_clause_en_1.0.0_3.0_1676303921297.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_termination_clause_en_1.0.0_3.0_1676303921297.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_termination_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[termination]| |[other]| |[other]| |[termination]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_termination_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 1.00 1.00 8 termination 1.00 1.00 1.00 14 accuracy - - 1.00 22 macro-avg 1.00 1.00 1.00 22 weighted-avg 1.00 1.00 1.00 22 ``` --- layout: model title: Part of Speech for Arabic author: John Snow Labs name: pos_ud_padt date: 2020-11-30 task: Part of Speech Tagging language: ar edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [pos, ar] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_padt_ar_2.7.0_2.4_1606721957579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_padt_ar_2.7.0_2.4_1606721957579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_padt", "ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate(["كرستيانو رونالدو لاعب برتغالي محترف يلعب في صفوف منتخب البرتغال لكرة القدم"]) ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_padt", "ar") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("كرستيانو رونالدو لاعب برتغالي محترف يلعب في صفوف منتخب البرتغال لكرة القدم").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""كرستيانو رونالدو لاعب برتغالي محترف يلعب في صفوف منتخب البرتغال لكرة القدم"""] pos_df = nlu.load('ar.pos').predict(text) pos_df ```

## Results ```bash {'pos': [Annotation(pos, 0, 7, X, {'word': 'كرستيانو'}), Annotation(pos, 9, 15, X, {'word': 'رونالدو'}), Annotation(pos, 17, 20, NOUN, {'word': 'لاعب'}), Annotation(pos, 22, 28, X, {'word': 'برتغالي'}), Annotation(pos, 30, 34, X, {'word': 'محترف'}), Annotation(pos, 36, 39, VERB, {'word': 'يلعب'}), Annotation(pos, 41, 42, ADP, {'word': 'في'}), Annotation(pos, 44, 47, NOUN, {'word': 'صفوف'}), Annotation(pos, 49, 53, NOUN, {'word': 'منتخب'}), Annotation(pos, 55, 62, X, {'word': 'البرتغال'}), Annotation(pos, 64, 67, CCONJ, {'word': 'لكرة'}), Annotation(pos, 69, 73, NOUN, {'word': 'القدم'})], ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_padt| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[tags, document]| |Output Labels:|[pos]| |Language:|ar| ## Data Source The model is trained on data obtained from [https://universaldependencies.org](https://universaldependencies.org) ## Benchmarking ```bash | | | precision | recall | f1-score | support | |---:|:-------------|:------------|:---------|-----------:|----------:| | 0 | ADJ | 0.90 | 0.91 | 0.91 | 2937 | | 1 | ADP | 0.99 | 1.00 | 0.99 | 4528 | | 2 | ADV | 0.96 | 0.93 | 0.95 | 104 | | 3 | AUX | 0.88 | 0.85 | 0.87 | 197 | | 4 | CCONJ | 1.00 | 0.99 | 0.99 | 1963 | | 5 | DET | 0.95 | 0.96 | 0.96 | 623 | | 6 | NOUN | 0.94 | 0.96 | 0.95 | 9547 | | 7 | NUM | 0.98 | 0.97 | 0.98 | 779 | | 8 | None | 1.00 | 1.00 | 1 | 3868 | | 9 | PART | 0.92 | 0.93 | 0.93 | 226 | | 10 | PRON | 0.99 | 1.00 | 1 | 1133 | | 11 | PROPN | 1.00 | 0.48 | 0.65 | 31 | | 12 | PUNCT | 1.00 | 1.00 | 1 | 2052 | | 13 | SCONJ | 0.99 | 0.98 | 0.98 | 534 | | 14 | SYM | 1.00 | 0.98 | 0.99 | 41 | | 15 | VERB | 0.94 | 0.93 | 0.94 | 2189 | | 16 | X | 0.80 | 0.64 | 0.71 | 1380 | | 17 | accuracy | | | 0.96 | 32132 | | 18 | macro avg | 0.95 | 0.91 | 0.93 | 32132 | | 19 | weighted avg | 0.95 | 0.96 | 0.95 | 32132 | ``` --- layout: model title: Translate Dutch to English Pipeline author: John Snow Labs name: translate_nl_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, nl, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `nl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_nl_en_xx_2.7.0_2.4_1609690917869.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_nl_en_xx_2.7.0_2.4_1609690917869.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_nl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_nl_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.nl.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_nl_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Split Sentences in Healthcare Texts author: John Snow Labs name: sentence_detector_dl_healthcare date: 2021-03-16 tags: [en, sentence_detection, licensed, clinical] task: Sentence Detection language: en nav_key: models edition: Healthcare NLP 2.7.0 spark_version: 2.4 supported: true annotator: SentenceDetectorDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SENTENCE_DETECTOR_HC/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/20.SentenceDetectorDL_Healthcare.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_2.7.0_2.4_1615880554391.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sentence_detector_dl_healthcare_en_2.7.0_2.4_1615880554391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl_healthcare","en","clinical/models") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) result = sd_model.fullAnnotate("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. LASIX CHANGED TO 40 PO BID WHICH IS SAME AS HE TAKES AT HOME - RECEIVED 40 PO IN AM - 700CC U/O TOTAL FOR FLUID NEGATIVE ~ 600 THUS FAR TODAY, ~ 600 NEG LOS.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","en","clinical/models") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. LASIX CHANGED TO 40 PO BID WHICH IS SAME AS HE TAKES AT HOME - RECEIVED 40 PO IN AM - 700CC U/O TOTAL FOR FLUID NEGATIVE ~ 600 THUS FAR TODAY, ~ 600 NEG LOS.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.detect_sentence.clinical").predict("""He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety.Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. LASIX CHANGED TO 40 PO BID WHICH IS SAME AS HE TAKES AT HOME - RECEIVED 40 PO IN AM - 700CC U/O TOTAL FOR FLUID NEGATIVE ~ 600 THUS FAR TODAY, ~ 600 NEG LOS.""") ```

## Results ```bash | | sentence | |---:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0 | He was given boluses of MS04 with some effect, he has since been placed on a PCA - he take 80mg of oxycontin at home, his PCA dose is ~ 2 the morphine dose of the oxycontin, he has also received ativan for anxiety. | | 1 | Repleted with 20 meq kcl po, 30 mmol K-phos iv and 2 gms mag so4 iv. | | 2 | LASIX CHANGED TO 40 PO BID WHICH IS SAME AS HE TAKES AT HOME - RECEIVED 40 PO IN AM - 700CC U/O TOTAL FOR FLUID NEGATIVE ~ 600 THUS FAR TODAY, ~ 600 NEG LOS. | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl_healthcare| |Compatibility:|Healthcare NLP 2.7.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|en| ## Data Source Healthcare SDDL model is trained on in-house domain specific data. ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English image_classifier_vit_base_patch16_224 ViTForImageClassification from google author: John Snow Labs name: image_classifier_vit_base_patch16_224 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224` is a English model originally trained by google. ## Predicted Entities `turnstile`, `damselfly`, `mixing bowl`, `sea snake`, `cockroach, roach`, `buckle`, `beer glass`, `bulbul`, `lumbermill, sawmill`, `whippet`, `Australian terrier`, `television, television system`, `hoopskirt, crinoline`, `horse cart, horse-cart`, `guillotine`, `malamute, malemute, Alaskan malamute`, `coyote, prairie wolf, brush wolf, Canis latrans`, `colobus, colobus monkey`, `hognose snake, puff adder, sand viper`, `sock`, `burrito`, `printer`, `bathing cap, swimming cap`, `chiton, coat-of-mail shell, sea cradle, polyplacophore`, `Rottweiler`, `cello, violoncello`, `pitcher, ewer`, `computer keyboard, keypad`, `bow`, `peacock`, `ballplayer, baseball player`, `refrigerator, icebox`, `solar dish, solar collector, solar furnace`, `passenger car, coach, carriage`, `African chameleon, Chamaeleo chamaeleon`, `oboe, hautboy, hautbois`, `toyshop`, `Leonberg`, `howler monkey, howler`, `bluetick`, `African elephant, Loxodonta africana`, `American lobster, Northern lobster, Maine lobster, Homarus americanus`, `combination lock`, `black-and-tan coonhound`, `bonnet, poke bonnet`, `harvester, reaper`, `Appenzeller`, `iron, smoothing iron`, `electric locomotive`, `lycaenid, lycaenid butterfly`, `sandbar, sand bar`, `Cardigan, Cardigan Welsh corgi`, `pencil sharpener`, `jean, blue jean, denim`, `backpack, back pack, knapsack, packsack, rucksack, haversack`, `monitor`, `ice cream, icecream`, `apiary, bee house`, `water jug`, `American coot, marsh hen, mud hen, water hen, Fulica americana`, `ground beetle, carabid beetle`, `jigsaw puzzle`, `ant, emmet, pismire`, `wreck`, `kuvasz`, `gyromitra`, `Ibizan hound, Ibizan Podenco`, `brown bear, bruin, Ursus arctos`, `bolo tie, bolo, bola tie, bola`, `Pembroke, Pembroke Welsh corgi`, `French bulldog`, `prison, prison house`, `ballpoint, ballpoint pen, ballpen, Biro`, `stage`, `airliner`, `dogsled, dog sled, dog sleigh`, `redshank, Tringa totanus`, `menu`, `Indian cobra, Naja naja`, `swab, swob, mop`, `window screen`, `brain coral`, `artichoke, globe artichoke`, `loupe, jeweler's loupe`, `loudspeaker, speaker, speaker unit, loudspeaker system, speaker system`, `panpipe, pandean pipe, syrinx`, `wok`, `croquet ball`, `plate`, `scoreboard`, `Samoyed, Samoyede`, `ocarina, sweet potato`, `beaver`, `borzoi, Russian wolfhound`, `horizontal bar, high bar`, `stretcher`, `seat belt, seatbelt`, `obelisk`, `forklift`, `feather boa, boa`, `frying pan, frypan, skillet`, `barbershop`, `hamper`, `face powder`, `Siamese cat, Siamese`, `ladle`, `dingo, warrigal, warragal, Canis dingo`, `mountain tent`, `head cabbage`, `echidna, spiny anteater, anteater`, `Polaroid camera, Polaroid Land camera`, `dumbbell`, `espresso`, `notebook, notebook computer`, `Norfolk terrier`, `binoculars, field glasses, opera glasses`, `carpenter's kit, tool kit`, `moving van`, `catamaran`, `tiger beetle`, `bikini, two-piece`, `Siberian husky`, `studio couch, day bed`, `bulletproof vest`, `lawn mower, mower`, `promontory, headland, head, foreland`, `soap dispenser`, `vulture`, `dam, dike, dyke`, `brambling, Fringilla montifringilla`, `toilet tissue, toilet paper, bathroom tissue`, `ringlet, ringlet butterfly`, `tiger cat`, `mobile home, manufactured home`, `Norwich terrier`, `little blue heron, Egretta caerulea`, `English setter`, `Tibetan mastiff`, `rocking chair, rocker`, `mask`, `maze, labyrinth`, `bookcase`, `viaduct`, `sweatshirt`, `plow, plough`, `basenji`, `typewriter keyboard`, `Windsor tie`, `coral fungus`, `desktop computer`, `Kerry blue terrier`, `Angora, Angora rabbit`, `can opener, tin opener`, `shield, buckler`, `triumphal arch`, `horned viper, cerastes, sand viper, horned asp, Cerastes cornutus`, `miniature schnauzer`, `tape player`, `jaguar, panther, Panthera onca, Felis onca`, `hook, claw`, `file, file cabinet, filing cabinet`, `chime, bell, gong`, `shower curtain`, `window shade`, `acoustic guitar`, `gas pump, gasoline pump, petrol pump, island dispenser`, `cicada, cicala`, `Petri dish`, `paintbrush`, `banana`, `chickadee`, `mountain bike, all-terrain bike, off-roader`, `lighter, light, igniter, ignitor`, `oil filter`, `cab, hack, taxi, taxicab`, `Christmas stocking`, `rugby ball`, `black widow, Latrodectus mactans`, `bustard`, `fiddler crab`, `web site, website, internet site, site`, `chocolate sauce, chocolate syrup`, `chainlink fence`, `fireboat`, `cocktail shaker`, `airship, dirigible`, `projectile, missile`, `bagel, beigel`, `screwdriver`, `oystercatcher, oyster catcher`, `pot, flowerpot`, `water bottle`, `Loafer`, `drumstick`, `soccer ball`, `cairn, cairn terrier`, `padlock`, `tow truck, tow car, wrecker`, `bloodhound, sleuthhound`, `punching bag, punch bag, punching ball, punchball`, `great grey owl, great gray owl, Strix nebulosa`, `scale, weighing machine`, `trench coat`, `briard`, `cheetah, chetah, Acinonyx jubatus`, `entertainment center`, `Boston bull, Boston terrier`, `Arabian camel, dromedary, Camelus dromedarius`, `steam locomotive`, `coil, spiral, volute, whorl, helix`, `plane, carpenter's plane, woodworking plane`, `gondola`, `spider web, spider's web`, `bathtub, bathing tub, bath, tub`, `pelican`, `miniature poodle`, `cowboy boot`, `perfume, essence`, `lakeside, lakeshore`, `timber wolf, grey wolf, gray wolf, Canis lupus`, `moped`, `sunscreen, sunblock, sun blocker`, `Brabancon griffon`, `puffer, pufferfish, blowfish, globefish`, `lifeboat`, `pool table, billiard table, snooker table`, `Bouvier des Flandres, Bouviers des Flandres`, `Pomeranian`, `theater curtain, theatre curtain`, `marimba, xylophone`, `baboon`, `vacuum, vacuum cleaner`, `pill bottle`, `pick, plectrum, plectron`, `hen`, `American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier`, `digital watch`, `pier`, `oxygen mask`, `Tibetan terrier, chrysanthemum dog`, `ostrich, Struthio camelus`, `water ouzel, dipper`, `drilling platform, offshore rig`, `magnetic compass`, `throne`, `butternut squash`, `minibus`, `EntleBucher`, `carousel, carrousel, merry-go-round, roundabout, whirligig`, `hot pot, hotpot`, `rain barrel`, `wood rabbit, cottontail, cottontail rabbit`, `miniature pinscher`, `partridge`, `three-toed sloth, ai, Bradypus tridactylus`, `English springer, English springer spaniel`, `corkscrew, bottle screw`, `fur coat`, `robin, American robin, Turdus migratorius`, `dowitcher`, `ruddy turnstone, Arenaria interpres`, `water snake`, `stove`, `Great Pyrenees`, `soft-coated wheaten terrier`, `carbonara`, `snail`, `breastplate, aegis, egis`, `wolf spider, hunting spider`, `hatchet`, `CD player`, `axolotl, mud puppy, Ambystoma mexicanum`, `pomegranate`, `poncho`, `leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea`, `lorikeet`, `spatula`, `jay`, `platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus`, `stethoscope`, `flagpole, flagstaff`, `coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch`, `agama`, `red wolf, maned wolf, Canis rufus, Canis niger`, `beaker`, `eft`, `pretzel`, `brassiere, bra, bandeau`, `frilled lizard, Chlamydosaurus kingi`, `joystick`, `goldfish, Carassius auratus`, `fig`, `maypole`, `caldron, cauldron`, `admiral`, `impala, Aepyceros melampus`, `spotted salamander, Ambystoma maculatum`, `syringe`, `hog, pig, grunter, squealer, Sus scrofa`, `handkerchief, hankie, hanky, hankey`, `tarantula`, `cheeseburger`, `pinwheel`, `sax, saxophone`, `dung beetle`, `broccoli`, `cassette player`, `milk can`, `traffic light, traffic signal, stoplight`, `shovel`, `sarong`, `tabby, tabby cat`, `parallel bars, bars`, `ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle`, `quill, quill pen`, `giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca`, `steel drum`, `quail`, `Blenheim spaniel`, `wig`, `hamster`, `ice lolly, lolly, lollipop, popsicle`, `seashore, coast, seacoast, sea-coast`, `chest`, `worm fence, snake fence, snake-rail fence, Virginia fence`, `missile`, `beer bottle`, `yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum`, `breakwater, groin, groyne, mole, bulwark, seawall, jetty`, `white wolf, Arctic wolf, Canis lupus tundrarum`, `guacamole`, `porcupine, hedgehog`, `trolleybus, trolley coach, trackless trolley`, `greenhouse, nursery, glasshouse`, `trimaran`, `Italian greyhound`, `potter's wheel`, `jacamar`, `wallet, billfold, notecase, pocketbook`, `Lakeland terrier`, `green lizard, Lacerta viridis`, `indigo bunting, indigo finch, indigo bird, Passerina cyanea`, `green mamba`, `walking stick, walkingstick, stick insect`, `crossword puzzle, crossword`, `eggnog`, `barrow, garden cart, lawn cart, wheelbarrow`, `remote control, remote`, `bicycle-built-for-two, tandem bicycle, tandem`, `wool, woolen, woollen`, `black grouse`, `abaya`, `marmoset`, `golf ball`, `jeep, landrover`, `Mexican hairless`, `dishwasher, dish washer, dishwashing machine`, `jersey, T-shirt, tee shirt`, `planetarium`, `goose`, `mailbox, letter box`, `capuchin, ringtail, Cebus capucinus`, `marmot`, `orangutan, orang, orangutang, Pongo pygmaeus`, `coffeepot`, `ambulance`, `shopping basket`, `pop bottle, soda bottle`, `red fox, Vulpes vulpes`, `crash helmet`, `street sign`, `affenpinscher, monkey pinscher, monkey dog`, `Arctic fox, white fox, Alopex lagopus`, `sidewinder, horned rattlesnake, Crotalus cerastes`, `ruffed grouse, partridge, Bonasa umbellus`, `muzzle`, `measuring cup`, `canoe`, `reflex camera`, `fox squirrel, eastern fox squirrel, Sciurus niger`, `French loaf`, `killer whale, killer, orca, grampus, sea wolf, Orcinus orca`, `dial telephone, dial phone`, `thimble`, `bubble`, `vizsla, Hungarian pointer`, `running shoe`, `mailbag, postbag`, `radio telescope, radio reflector`, `piggy bank, penny bank`, `Chihuahua`, `chambered nautilus, pearly nautilus, nautilus`, `Airedale, Airedale terrier`, `kimono`, `green snake, grass snake`, `rubber eraser, rubber, pencil eraser`, `upright, upright piano`, `orange`, `revolver, six-gun, six-shooter`, `ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin`, `drum, membranophone, tympan`, `Dungeness crab, Cancer magister`, `lipstick, lip rouge`, `gong, tam-tam`, `fountain`, `tub, vat`, `malinois`, `sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita`, `German short-haired pointer`, `apron`, `Irish setter, red setter`, `dishrag, dishcloth`, `school bus`, `candle, taper, wax light`, `bib`, `cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM`, `power drill`, `English foxhound`, `miniskirt, mini`, `swing`, `slug`, `hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa`, `rifle`, `Saluki, gazelle hound`, `Sealyham terrier, Sealyham`, `bullet train, bullet`, `hyena, hyaena`, `ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus`, `toy terrier`, `goblet`, `safe`, `cup`, `electric guitar`, `red wine`, `restaurant, eating house, eating place, eatery`, `wall clock`, `washbasin, handbasin, washbowl, lavabo, wash-hand basin`, `red-breasted merganser, Mergus serrator`, `crate`, `banded gecko`, `hippopotamus, hippo, river horse, Hippopotamus amphibius`, `tick`, `tripod`, `sombrero`, `desk`, `sea slug, nudibranch`, `racer, race car, racing car`, `pizza, pizza pie`, `dining table, board`, `Saint Bernard, St Bernard`, `komondor`, `electric ray, crampfish, numbfish, torpedo`, `prairie chicken, prairie grouse, prairie fowl`, `coffee mug`, `hammer`, `golfcart, golf cart`, `unicycle, monocycle`, `bison`, `soup bowl`, `rapeseed`, `golden retriever`, `plastic bag`, `grey fox, gray fox, Urocyon cinereoargenteus`, `water tower`, `house finch, linnet, Carpodacus mexicanus`, `barbell`, `hair slide`, `tiger, Panthera tigris`, `black-footed ferret, ferret, Mustela nigripes`, `meat loaf, meatloaf`, `hand blower, blow dryer, blow drier, hair dryer, hair drier`, `overskirt`, `gibbon, Hylobates lar`, `Gila monster, Heloderma suspectum`, `toucan`, `snowmobile`, `pencil box, pencil case`, `scuba diver`, `cloak`, `Sussex spaniel`, `otter`, `Greater Swiss Mountain dog`, `great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias`, `torch`, `magpie`, `tiger shark, Galeocerdo cuvieri`, `wing`, `Border collie`, `bell cote, bell cot`, `sea anemone, anemone`, `teapot`, `sea urchin`, `screen, CRT screen`, `bookshop, bookstore, bookstall`, `oscilloscope, scope, cathode-ray oscilloscope, CRO`, `crib, cot`, `police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria`, `hartebeest`, `manhole cover`, `iPod`, `rock python, rock snake, Python sebae`, `nipple`, `suspension bridge`, `safety pin`, `sea lion`, `cougar, puma, catamount, mountain lion, painter, panther, Felis concolor`, `mantis, mantid`, `wardrobe, closet, press`, `projector`, `Granny Smith`, `diamondback, diamondback rattlesnake, Crotalus adamanteus`, `pirate, pirate ship`, `espresso maker`, `African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus`, `cradle`, `common newt, Triturus vulgaris`, `tricycle, trike, velocipede`, `bobsled, bobsleigh, bob`, `thunder snake, worm snake, Carphophis amoenus`, `thresher, thrasher, threshing machine`, `banjo`, `armadillo`, `pajama, pyjama, pj's, jammies`, `ski`, `Maltese dog, Maltese terrier, Maltese`, `leafhopper`, `book jacket, dust cover, dust jacket, dust wrapper`, `silky terrier, Sydney silky`, `Shih-Tzu`, `wallaby, brush kangaroo`, `cardigan`, `sturgeon`, `freight car`, `home theater, home theatre`, `sundial`, `African crocodile, Nile crocodile, Crocodylus niloticus`, `odometer, hodometer, mileometer, milometer`, `sliding door`, `vine snake`, `West Highland white terrier`, `mongoose`, `hornbill`, `beagle`, `European gallinule, Porphyrio porphyrio`, `submarine, pigboat, sub, U-boat`, `Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis`, `cock`, `pedestal, plinth, footstall`, `accordion, piano accordion, squeeze box`, `gown`, `lynx, catamount`, `guenon, guenon monkey`, `Walker hound, Walker foxhound`, `standard schnauzer`, `reel`, `hip, rose hip, rosehip`, `grasshopper, hopper`, `Dutch oven`, `stone wall`, `hard disc, hard disk, fixed disk`, `snow leopard, ounce, Panthera uncia`, `shopping cart`, `digital clock`, `hourglass`, `Border terrier`, `Old English sheepdog, bobtail`, `academic gown, academic robe, judge's robe`, `spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish`, `spotlight, spot`, `dome`, `barn spider, Araneus cavaticus`, `bee eater`, `basketball`, `cliff dwelling`, `folding chair`, `isopod`, `Doberman, Doberman pinscher`, `bittern`, `sunglasses, dark glasses, shades`, `picket fence, paling`, `Crock Pot`, `ibex, Capra ibex`, `neck brace`, `cardoon`, `cassette`, `amphibian, amphibious vehicle`, `minivan`, `analog clock`, `trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi`, `yurt`, `cliff, drop, drop-off`, `Bernese mountain dog`, `teddy, teddy bear`, `sloth bear, Melursus ursinus, Ursus ursinus`, `bassoon`, `toaster`, `ptarmigan`, `Gordon setter`, `night snake, Hypsiglena torquata`, `grand piano, grand`, `purse`, `clumber, clumber spaniel`, `shoji`, `hair spray`, `maillot`, `knee pad`, `space heater`, `bottlecap`, `chiffonier, commode`, `chain saw, chainsaw`, `sulphur butterfly, sulfur butterfly`, `pay-phone, pay-station`, `kelpie`, `mouse, computer mouse`, `car wheel`, `cornet, horn, trumpet, trump`, `container ship, containership, container vessel`, `matchstick`, `scabbard`, `American black bear, black bear, Ursus americanus, Euarctos americanus`, `langur`, `rock crab, Cancer irroratus`, `lionfish`, `speedboat`, `black stork, Ciconia nigra`, `knot`, `disk brake, disc brake`, `mosquito net`, `white stork, Ciconia ciconia`, `abacus`, `titi, titi monkey`, `grocery store, grocery, food market, market`, `waffle iron`, `pickelhaube`, `wooden spoon`, `Norwegian elkhound, elkhound`, `earthstar`, `sewing machine`, `balance beam, beam`, `potpie`, `chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour`, `Staffordshire bullterrier, Staffordshire bull terrier`, `switch, electric switch, electrical switch`, `dhole, Cuon alpinus`, `paddle, boat paddle`, `limousine, limo`, `Shetland sheepdog, Shetland sheep dog, Shetland`, `space bar`, `library`, `paddlewheel, paddle wheel`, `alligator lizard`, `Band Aid`, `Persian cat`, `bull mastiff`, `tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui`, `sports car, sport car`, `football helmet`, `laptop, laptop computer`, `lens cap, lens cover`, `tennis ball`, `violin, fiddle`, `lab coat, laboratory coat`, `cinema, movie theater, movie theatre, movie house, picture palace`, `weasel`, `bow tie, bow-tie, bowtie`, `macaw`, `dough`, `whiskey jug`, `microphone, mike`, `spoonbill`, `bassinet`, `mud turtle`, `velvet`, `warthog`, `plunger, plumber's helper`, `dugong, Dugong dugon`, `honeycomb`, `badger`, `dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk`, `bee`, `doormat, welcome mat`, `fountain pen`, `giant schnauzer`, `assault rifle, assault gun`, `limpkin, Aramus pictus`, `siamang, Hylobates syndactylus, Symphalangus syndactylus`, `albatross, mollymawk`, `confectionery, confectionary, candy store`, `harp`, `parachute, chute`, `barrel, cask`, `tank, army tank, armored combat vehicle, armoured combat vehicle`, `collie`, `kite`, `puck, hockey puck`, `stupa, tope`, `buckeye, horse chestnut, conker`, `patio, terrace`, `broom`, `Dandie Dinmont, Dandie Dinmont terrier`, `scorpion`, `agaric`, `balloon`, `bucket, pail`, `squirrel monkey, Saimiri sciureus`, `Eskimo dog, husky`, `zebra`, `garter snake, grass snake`, `indri, indris, Indri indri, Indri brevicaudatus`, `tractor`, `guinea pig, Cavia cobaya`, `maraca`, `red-backed sandpiper, dunlin, Erolia alpina`, `bullfrog, Rana catesbeiana`, `trilobite`, `Japanese spaniel`, `gorilla, Gorilla gorilla`, `monastery`, `centipede`, `terrapin`, `llama`, `long-horned beetle, longicorn, longicorn beetle`, `boxer`, `curly-coated retriever`, `mortar`, `hammerhead, hammerhead shark`, `goldfinch, Carduelis carduelis`, `garden spider, Aranea diademata`, `stopwatch, stop watch`, `grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus`, `leaf beetle, chrysomelid`, `birdhouse`, `king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica`, `stole`, `bell pepper`, `radiator`, `flatworm, platyhelminth`, `mushroom`, `Scotch terrier, Scottish terrier, Scottie`, `liner, ocean liner`, `toilet seat`, `lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens`, `zucchini, courgette`, `harvestman, daddy longlegs, Phalangium opilio`, `Newfoundland, Newfoundland dog`, `flamingo`, `whiptail, whiptail lizard`, `geyser`, `cleaver, meat cleaver, chopper`, `sea cucumber, holothurian`, `American egret, great white heron, Egretta albus`, `parking meter`, `beacon, lighthouse, beacon light, pharos`, `coucal`, `motor scooter, scooter`, `mitten`, `cannon`, `weevil`, `megalith, megalithic structure`, `stinkhorn, carrion fungus`, `ear, spike, capitulum`, `box turtle, box tortoise`, `snowplow, snowplough`, `tench, Tinca tinca`, `modem`, `tobacco shop, tobacconist shop, tobacconist`, `barn`, `skunk, polecat, wood pussy`, `African grey, African gray, Psittacus erithacus`, `Madagascar cat, ring-tailed lemur, Lemur catta`, `holster`, `barometer`, `sleeping bag`, `washer, automatic washer, washing machine`, `recreational vehicle, RV, R.V.`, `drake`, `tray`, `butcher shop, meat market`, `china cabinet, china closet`, `medicine chest, medicine cabinet`, `photocopier`, `Yorkshire terrier`, `starfish, sea star`, `racket, racquet`, `park bench`, `Labrador retriever`, `whistle`, `clog, geta, patten, sabot`, `volcano`, `quilt, comforter, comfort, puff`, `leopard, Panthera pardus`, `cauliflower`, `swimming trunks, bathing trunks`, `American chameleon, anole, Anolis carolinensis`, `alp`, `mortarboard`, `barracouta, snoek`, `cocker spaniel, English cocker spaniel, cocker`, `space shuttle`, `beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon`, `harmonica, mouth organ, harp, mouth harp`, `gasmask, respirator, gas helmet`, `wombat`, `Model T`, `wild boar, boar, Sus scrofa`, `hermit crab`, `flat-coated retriever`, `rotisserie`, `jinrikisha, ricksha, rickshaw`, `trifle`, `bannister, banister, balustrade, balusters, handrail`, `go-kart`, `bakery, bakeshop, bakehouse`, `ski mask`, `dock, dockage, docking facility`, `Egyptian cat`, `oxcart`, `redbone`, `shoe shop, shoe-shop, shoe store`, `convertible`, `ox`, `crayfish, crawfish, crawdad, crawdaddy`, `cowboy hat, ten-gallon hat`, `conch`, `spaghetti squash`, `toy poodle`, `saltshaker, salt shaker`, `microwave, microwave oven`, `triceratops`, `necklace`, `castle`, `streetcar, tram, tramcar, trolley, trolley car`, `eel`, `diaper, nappy, napkin`, `standard poodle`, `prayer rug, prayer mat`, `radio, wireless`, `crane`, `envelope`, `rule, ruler`, `gar, garfish, garpike, billfish, Lepisosteus osseus`, `spider monkey, Ateles geoffroyi`, `Irish wolfhound`, `German shepherd, German shepherd dog, German police dog, alsatian`, `umbrella`, `sunglass`, `aircraft carrier, carrier, flattop, attack aircraft carrier`, `water buffalo, water ox, Asiatic buffalo, Bubalus bubalis`, `jellyfish`, `groom, bridegroom`, `tree frog, tree-frog`, `steel arch bridge`, `lemon`, `pickup, pickup truck`, `vault`, `groenendael`, `baseball`, `junco, snowbird`, `maillot, tank suit`, `gazelle`, `jack-o'-lantern`, `military uniform`, `slide rule, slipstick`, `wire-haired fox terrier`, `acorn squash`, `electric fan, blower`, `Brittany spaniel`, `chimpanzee, chimp, Pan troglodytes`, `pillow`, `binder, ring-binder`, `schipperke`, `Afghan hound, Afghan`, `plate rack`, `car mirror`, `hand-held computer, hand-held microcomputer`, `papillon`, `schooner`, `Bedlington terrier`, `cellular telephone, cellular phone, cellphone, cell, mobile phone`, `altar`, `Chesapeake Bay retriever`, `cabbage butterfly`, `polecat, fitch, foulmart, foumart, Mustela putorius`, `comic book`, `French horn, horn`, `daisy`, `organ, pipe organ`, `mashed potato`, `acorn`, `fly`, `chain`, `American alligator, Alligator mississipiensis`, `mink`, `garbage truck, dustcart`, `totem pole`, `wine bottle`, `strawberry`, `cricket`, `European fire salamander, Salamandra salamandra`, `coral reef`, `Welsh springer spaniel`, `bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis`, `snorkel`, `bald eagle, American eagle, Haliaeetus leucocephalus`, `meerkat, mierkat`, `grille, radiator grille`, `nematode, nematode worm, roundworm`, `anemone fish`, `corn`, `loggerhead, loggerhead turtle, Caretta caretta`, `palace`, `suit, suit of clothes`, `pineapple, ananas`, `macaque`, `ping-pong ball`, `ram, tup`, `church, church building`, `koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus`, `hare`, `bath towel`, `strainer`, `yawl`, `otterhound, otter hound`, `table lamp`, `king snake, kingsnake`, `lotion`, `lion, king of beasts, Panthera leo`, `thatch, thatched roof`, `basset, basset hound`, `black and gold garden spider, Argiope aurantia`, `barber chair`, `proboscis monkey, Nasalis larvatus`, `consomme`, `Irish terrier`, `Irish water spaniel`, `common iguana, iguana, Iguana iguana`, `Weimaraner`, `Great Dane`, `pug, pug-dog`, `rhinoceros beetle`, `vase`, `brass, memorial tablet, plaque`, `kit fox, Vulpes macrotis`, `king penguin, Aptenodytes patagonica`, `vending machine`, `dalmatian, coach dog, carriage dog`, `rock beauty, Holocanthus tricolor`, `pole`, `cuirass`, `bolete`, `jackfruit, jak, jack`, `monarch, monarch butterfly, milkweed butterfly, Danaus plexippus`, `chow, chow chow`, `nail`, `packet`, `half track`, `Lhasa, Lhasa apso`, `boathouse`, `hay`, `valley, vale`, `slot, one-armed bandit`, `volleyball`, `carton`, `shower cap`, `tile roof`, `lacewing, lacewing fly`, `patas, hussar monkey, Erythrocebus patas`, `boa constrictor, Constrictor constrictor`, `black swan, Cygnus atratus`, `lampshade, lamp shade`, `mousetrap`, `crutch`, `vestment`, `Pekinese, Pekingese, Peke`, `tusker`, `warplane, military plane`, `sandal`, `screw`, `custard apple`, `Scottish deerhound, deerhound`, `spindle`, `keeshond`, `hummingbird`, `letter opener, paper knife, paperknife`, `cucumber, cuke`, `bearskin, busby, shako`, `fire engine, fire truck`, `trombone`, `ringneck snake, ring-necked snake, ring snake`, `sorrel`, `fire screen, fireguard`, `paper towel`, `flute, transverse flute`, `hotdog, hot dog, red hot`, `Indian elephant, Elephas maximus`, `mosque`, `stingray`, `Rhodesian ridgeback`, `four-poster` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_en_4.1.0_3.0_1660165449585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_en_4.1.0_3.0_1660165449585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|324.8 MB| --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from ms12345) author: John Snow Labs name: distilbert_qa_ms12345_base_cased_led_squad_finetuned date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-finetuned-squad` is a English model originally trained by `ms12345`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ms12345_base_cased_led_squad_finetuned_en_4.3.0_3.0_1672766560316.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ms12345_base_cased_led_squad_finetuned_en_4.3.0_3.0_1672766560316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ms12345_base_cased_led_squad_finetuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ms12345_base_cased_led_squad_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ms12345_base_cased_led_squad_finetuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ms12345/distilbert-base-cased-distilled-squad-finetuned-squad --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from karukapur) author: John Snow Labs name: distilbert_qa_karukapur_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `karukapur`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_karukapur_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771710627.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_karukapur_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771710627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_karukapur_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_karukapur_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_karukapur_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/karukapur/distilbert-base-uncased-finetuned-squad --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dl2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dl2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl2_en_4.3.0_3.0_1675109902657.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dl2_en_4.3.0_3.0_1675109902657.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|293.7 MB| ## References - https://huggingface.co/google/t5-efficient-base-dl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Question Pair Classifier author: John Snow Labs name: classifierdl_electra_questionpair date: 2021-08-13 tags: [en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.1.3 spark_version: 2.4 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Identifies whether two question sentences are semantically repetitive or different. ## Predicted Entities `almost_same`, `not_same`. {:.btn-box} {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_QUESTIONPAIR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_QUESTIONPAIRS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_electra_questionpair_en_3.1.3_2.4_1628840750568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_electra_questionpair_en_3.1.3_2.4_1628840750568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use - The model is trained with `sent_electra_large_uncased` embeddings therefore the same embeddings should be used in the prediction pipeline. - The question pairs should be identified with "q1" and "q2" in the text. The input text format should be as follows : `text = "q1: What is your name? q2: Who are you?"`

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = BertSentenceEmbeddings.pretrained("sent_electra_large_uncased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") document_classifier = ClassifierDLModel.pretrained('classifierdl_electra_questionpair', 'en') \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") nlpPipeline = Pipeline(stages=[document, embeddings, document_classifier]) light_pipeline = LightPipeline(nlpPipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result_1 = light_pipeline.annotate("q1: What is your favorite movie? q2: Which movie do you like most?") print(result_1["class"]) result_2 = light_pipeline.annotate("q1: What is your favorite movie? q2: Which movie genre would you like to watch?") print(result_2["class"]) ``` ```scala val document = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val embeddings = BertSentenceEmbeddings.pretrained("sent_electra_large_uncased", "en") .setInputCols("document") .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_electra_questionpair", 'en') .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val nlpPipeline = new Pipeline().setStages(Array(document, embeddings, document_classifier)) val light_pipeline = LightPipeline(nlpPipeline.fit(spark.createDataFrame([['']]).toDF("text"))) val result_1 = light_pipeline.annotate("q1: What is your favorite movie? q2: Which movie do you like most?") val result_2 = light_pipeline.annotate("q1: What is your favorite movie? q2: Which movie genre would you like to watch?") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.questionpair").predict("""q1: What is your favorite movie? q2: Which movie genre would you like to watch?""") ```

## Results ```bash ['almost_same'] ['not_same'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_electra_questionpair| |Compatibility:|Spark NLP 3.1.3+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| ## Data Source A custom dataset is used based on this source : "https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs". ## Benchmarking ```bash label precision recall f1-score support almost_same 0.85 0.91 0.88 29652 not_same 0.90 0.84 0.87 29634 accuracy - - 0.88 59286 macro-avg 0.88 0.88 0.88 59286 weighted-avg 0.88 0.88 0.88 59286 ``` --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543_de_4.2.0_3.0_1664118461374.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543_de_4.2.0_3.0_1664118461374.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s543| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Abkhazian asr_xls_r_ab_test_by_muneson TFWav2Vec2ForCTC from muneson author: John Snow Labs name: asr_xls_r_ab_test_by_muneson date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_muneson` is a Abkhazian model originally trained by muneson. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_ab_test_by_muneson_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_muneson_ab_4.2.0_3.0_1664019191275.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_muneson_ab_4.2.0_3.0_1664019191275.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_ab_test_by_muneson", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_ab_test_by_muneson", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_ab_test_by_muneson| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|446.5 KB| --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0_en_4.0.0_3.0_1657191925822.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0_en_4.0.0_3.0_1657191925822.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|375.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-0 --- layout: model title: Sentence Embeddings - sbiobert (tuned) author: John Snow Labs name: sbiobert_jsl_cased date: 2021-05-14 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_cased_en_3.0.3_2.4_1621017156951.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobert_jsl_cased_en_3.0.3_2.4_1621017156951.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings\ .pretrained("sbiobert_jsl_cased","en","clinical/models")\ .setInputCols(["sentence"])\ .setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbiobert_jsl_cased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.biobert.jsl_cased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobert_jsl_cased| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Tuned on MedNLI and UMLS dataset ## Benchmarking ```bash MedNLI Score Acc 0.788 STS(cos) 0.736 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_en_4.3.0_3.0_1675120502722.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el8_en_4.3.0_3.0_1675120502722.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|159.9 MB| ## References - https://huggingface.co/google/t5-efficient-small-el8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from mrm8488) author: John Snow Labs name: t5_small_finetuned_emotion date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-finetuned-emotion` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_emotion_en_4.3.0_3.0_1675125983774.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_emotion_en_4.3.0_3.0_1675125983774.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_finetuned_emotion","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_finetuned_emotion","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_finetuned_emotion| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|263.3 MB| ## References - https://huggingface.co/mrm8488/t5-small-finetuned-emotion - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://github.com/dair-ai/emotion_dataset - https://arxiv.org/pdf/1910.10683.pdf - https://i.imgur.com/jVFMMWR.png - https://twitter.com/omarsar0 - https://github.com/dair-ai/emotion_dataset - https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb - https://github.com/patil-suraj - https://i.imgur.com/JBtAwPx.png - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: Question classification of open-domain and fact-based questions Pipeline - TREC50 author: John Snow Labs name: classifierdl_use_trec50_pipeline date: 2021-01-08 task: [Text Classification, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [classifier, text_classification, pipeline, en, open_source] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify open-domain, fact-based questions into one of the following broad semantic categories: Abbreviation, Description, Entities, Human Beings, Locations or Numeric Values. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_EN_TREC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_EN_TREC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec50_pipeline_en_2.7.1_2.4_1610119592993.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_trec50_pipeline_en_2.7.1_2.4_1610119592993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("classifierdl_use_trec50_pipeline", lang = "en") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("classifierdl_use_trec50_pipeline", lang = "en") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.trec50.component_list").predict("""Put your text here.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_dl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_dl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_dl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|54.2 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-dl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: NER Pipeline for 6 Scandinavian Languages author: John Snow Labs name: bert_token_classifier_scandi_ner_pipeline date: 2022-06-25 tags: [danish, norwegian, swedish, icelandic, faroese, bert, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on [bert_token_classifier_scandi_ner](https://nlp.johnsnowlabs.com/2021/12/09/bert_token_classifier_scandi_ner_xx.html) model which is imported from `HuggingFace`. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_SCANDINAVIAN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_scandi_ner_pipeline_xx_4.0.0_3.0_1656119754783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_scandi_ner_pipeline_xx_4.0.0_3.0_1656119754783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python scandiner_pipeline = PretrainedPipeline("bert_token_classifier_scandi_ner_pipeline", lang = "xx") scandiner_pipeline.annotate("Hans er professor ved Statens Universitet, som ligger i København, og han er en rigtig københavner.") ``` ```scala val scandiner_pipeline = new PretrainedPipeline("bert_token_classifier_scandi_ner_pipeline", lang = "xx") val scandiner_pipeline.annotate("Hans er professor ved Statens Universitet, som ligger i København, og han er en rigtig københavner.") ```

## Results ```bash +-------------------+---------+ |chunk |ner_label| +-------------------+---------+ |Hans |PER | |Statens Universitet|ORG | |København |LOC | |københavner |MISC | +-------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_scandi_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|xx| |Size:|666.9 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - BertForTokenClassification - NerConverter - Finisher --- layout: model title: Detect Cellular/Molecular Biology Entities author: John Snow Labs name: bert_token_classifier_ner_jnlpba_cellular date: 2022-07-25 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects molecular biology-related terms in medical texts. The model is trained with the BertForTokenClassification method from the transformers library and imported into Spark NLP. ## Predicted Entities `cell_line`, `cell_type`, `protein`, `DNA`, `RNA` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jnlpba_cellular_en_4.0.0_3.0_1658754953153.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jnlpba_cellular_en_4.0.0_3.0_1658754953153.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_jnlpba_cellular", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, ner_model, ner_converter ]) data = spark.createDataFrame([["""The results suggest that activation of protein kinase C, but not new protein synthesis, is required for IL-2 induction of IFN-gamma and GM-CSF cytoplasmic mRNA. It also was observed that suppression of cytokine gene expression by these agents was independent of the inhibition of proliferation. These data indicate that IL-2 and IL-12 may have distinct signaling pathways leading to the induction of IFN-gammaand GM-CSFgene expression, andthatthe NK3.3 cell line may serve as a novel model for dissecting the biochemical and molecular events involved in these pathways. A functional T-cell receptor signaling pathway is required for p95vav activity. Stimulation of the T-cell antigen receptor ( TCR ) induces activation of multiple tyrosine kinases, resulting in phosphorylation of numerous intracellular substrates. One substrate is p95vav, which is expressed exclusively in hematopoietic and trophoblast cells."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare", "en", "clinical/models") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val ner_model = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_jnlpba_cellular", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("ner") .setCaseSensitive(True) .setMaxSentenceLength(512) val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, ner_model, ner_converter)) val data = Seq("""The results suggest that activation of protein kinase C, but not new protein synthesis, is required for IL-2 induction of IFN-gamma and GM-CSF cytoplasmic mRNA. It also was observed that suppression of cytokine gene expression by these agents was independent of the inhibition of proliferation. These data indicate that IL-2 and IL-12 may have distinct signaling pathways leading to the induction of IFN-gammaand GM-CSFgene expression, andthatthe NK3.3 cell line may serve as a novel model for dissecting the biochemical and molecular events involved in these pathways. A functional T-cell receptor signaling pathway is required for p95vav activity. Stimulation of the T-cell antigen receptor ( TCR ) induces activation of multiple tyrosine kinases, resulting in phosphorylation of numerous intracellular substrates. One substrate is p95vav, which is expressed exclusively in hematopoietic and trophoblast cells.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.jnlpba_cellular").predict("""The results suggest that activation of protein kinase C, but not new protein synthesis, is required for IL-2 induction of IFN-gamma and GM-CSF cytoplasmic mRNA. It also was observed that suppression of cytokine gene expression by these agents was independent of the inhibition of proliferation. These data indicate that IL-2 and IL-12 may have distinct signaling pathways leading to the induction of IFN-gammaand GM-CSFgene expression, andthatthe NK3.3 cell line may serve as a novel model for dissecting the biochemical and molecular events involved in these pathways. A functional T-cell receptor signaling pathway is required for p95vav activity. Stimulation of the T-cell antigen receptor ( TCR ) induces activation of multiple tyrosine kinases, resulting in phosphorylation of numerous intracellular substrates. One substrate is p95vav, which is expressed exclusively in hematopoietic and trophoblast cells.""") ```

## Results ```bash +-------------------------------------+---------+ |ner_chunk |label | +-------------------------------------+---------+ |protein kinase C |protein | |IL-2 |protein | |IFN-gamma and GM-CSF cytoplasmic mRNA|RNA | |cytokine gene |DNA | |IL-2 |protein | |IL-12 |protein | |IFN-gammaand GM-CSFgene |protein | |NK3.3 cell line |cell_line| |T-cell receptor |protein | |p95vav |protein | |T-cell antigen receptor |protein | |TCR |protein | |tyrosine kinases |protein | |p95vav |protein | |hematopoietic and trophoblast cells |cell_type| +-------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_jnlpba_cellular| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://github.com/cambridgeltl/MTL-Bioinformatics-2016](https://github.com/cambridgeltl/MTL-Bioinformatics-2016) ## Benchmarking ```bash label precision recall f1-score support B-cell_line 0.5850 0.6880 0.6324 500 I-cell_line 0.6374 0.7644 0.6952 989 B-DNA 0.7187 0.7453 0.7318 1056 I-DNA 0.8134 0.8603 0.8362 1789 B-protein 0.7286 0.8429 0.7816 5067 I-protein 0.8020 0.8129 0.8074 4774 B-RNA 0.6812 0.7966 0.7344 118 I-RNA 0.8358 0.8984 0.8660 187 B-cell_type 0.7768 0.7501 0.7632 1921 I-cell_type 0.8654 0.7887 0.8253 2991 micro-avg 0.7673 0.8065 0.7864 19392 macro-avg 0.7444 0.7948 0.7673 19392 weighted-avg 0.7722 0.8065 0.7875 19392 ``` --- layout: model title: Arabic Bert Embeddings (Base, 1790k Iterations) author: John Snow Labs name: bert_embeddings_bert_base_qarib60_1790k date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-qarib60_1790k` is a Arabic model orginally trained by `qarib`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_qarib60_1790k_ar_3.4.2_3.0_1649678882711.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_qarib60_1790k_ar_3.4.2_3.0_1649678882711.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_qarib60_1790k| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|507.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/qarib/bert-base-qarib60_1790k - http://opus.nlpl.eu/ - https://github.com/qcri/QARIB/Training_QARiB.md - https://github.com/qcri/QARIB/Using_QARiB.md --- layout: model title: Fast Neural Machine Translation Model from English to Eastern Malayo-Polynesian Languages author: John Snow Labs name: opus_mt_en_pqe date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, pqe, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `pqe` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_pqe_xx_2.7.0_2.4_1609164776742.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_pqe_xx_2.7.0_2.4_1609164776742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_pqe| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from it5) author: John Snow Labs name: t5_it5_efficient_small_el32_ilgiornale_to_repubblica date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-el32-ilgiornale-to-repubblica` is a Italian model originally trained by `it5`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_ilgiornale_to_repubblica_it_4.3.0_3.0_1675103353037.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_ilgiornale_to_repubblica_it_4.3.0_3.0_1675103353037.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_ilgiornale_to_repubblica","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_ilgiornale_to_repubblica","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_el32_ilgiornale_to_repubblica| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|593.9 MB| ## References - https://huggingface.co/it5/it5-efficient-small-el32-ilgiornale-to-repubblica - https://github.com/stefan-it - https://arxiv.org/abs/2203.03759 - https://gsarti.com - https://malvinanissim.github.io - https://arxiv.org/abs/2109.10686 - https://github.com/gsarti/it5 - https://paperswithcode.com/sota?task=Headline+style+transfer+%28Il+Giornale+to+Repubblica%29&dataset=CHANGE-IT --- layout: model title: Persian XlmRoBertaForQuestionAnswering model (from m3hrdadfi) author: John Snow Labs name: xlmroberta_qa_xlmr_large date: 2022-06-28 tags: [fa, open_source, xlmroberta, question_answering] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlmr-large-qa-fa` is a Persian model originally trained by `m3hrdadfi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_qa_xlmr_large_fa_4.0.0_3.0_1656419067604.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_qa_xlmr_large_fa_4.0.0_3.0_1656419067604.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlmroberta_qa_xlmr_large","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = XlmRoBertaForQuestionAnswering.pretrained("xlmroberta_qa_xlmr_large","fa") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("fa.answer_question.xlmr_roberta.large").predict("""اسم من چیست؟|||"نام من کلارا است و من در برکلی زندگی می کنم.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_qa_xlmr_large| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|fa| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/m3hrdadfi/xlmr-large-qa-fa - https://github.com/sajjjadayobi/PersianQA - https://github.com/m3hrdadfi --- layout: model title: Detect Clinical Entities (bert_token_classifier_ner_clinical) author: John Snow Labs name: bert_token_classifier_ner_clinical date: 2022-01-06 tags: [berfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a BERT-based version of the ner_clinical model and it is 4% better than the legacy NER model (MedicalNerModel) that is based on BiLSTM-CNN-Char architecture. ## Predicted Entities `PROBLEM`, `TEST`, `TREATMENT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_en_3.3.4_2.4_1641472941908.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_en_3.3.4_2.4_1641472941908.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","en","clinical/models")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_ner_clinical", "en", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("ner") pipeline = Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, tokenClassifier ]) p_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) text = ''' A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge . ''' result = p_model.transform(spark.createDataFrame([[text]]).toDF("text")) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl_healthcare","en","clinical/models") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ner_clinical", "en", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler,sentenceDetector,tokenizer,tokenClassifier)) val data = Seq("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge .""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.ner_clinical").predict(""" A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . Two weeks prior to presentation , she was treated with a five-day course of amoxicillin for a respiratory tract infection . She was on metformin , glipizide , and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG . She had been on dapagliflozin for six months at the time of presentation . Physical examination on presentation was significant for dry oral mucosa ; significantly , her abdominal examination was benign with no tenderness , guarding , or rigidity . Pertinent laboratory findings on admission were : serum glucose 111 mg/dl , bicarbonate 18 mmol/l , anion gap 20 , creatinine 0.4 mg/dL , triglycerides 508 mg/dL , total cholesterol 122 mg/dL , glycated hemoglobin ( HbA1c ) 10% , and venous pH 7.27 . Serum lipase was normal at 43 U/L . Serum acetone levels could not be assessed as blood samples kept hemolyzing due to significant lipemia . The patient was initially admitted for starvation ketosis , as she reported poor oral intake for three days prior to admission . However , serum chemistry obtained six hours after presentation revealed her glucose was 186 mg/dL , the anion gap was still elevated at 21 , serum bicarbonate was 16 mmol/L , triglyceride level peaked at 2050 mg/dL , and lipase was 52 U/L . The β-hydroxybutyrate level was obtained and found to be elevated at 5.29 mmol/L - the original sample was centrifuged and the chylomicron layer removed prior to analysis due to interference from turbidity caused by lipemia again . The patient was treated with an insulin drip for euDKA and HTG with a reduction in the anion gap to 13 and triglycerides to 1400 mg/dL , within 24 hours . Her euDKA was thought to be precipitated by her respiratory tract infection in the setting of SGLT2 inhibitor use . The patient was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , and metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely . She had close follow-up with endocrinology post discharge . """) ```

## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |gestational diabetes mellitus |PROBLEM | |type two diabetes mellitus |PROBLEM | |T2DM |PROBLEM | |HTG-induced pancreatitis |PROBLEM | |an acute hepatitis |PROBLEM | |obesity |PROBLEM | |a body mass index |TEST | |BMI |TEST | |polyuria |PROBLEM | |polydipsia |PROBLEM | |poor appetite |PROBLEM | |vomiting |PROBLEM | |amoxicillin |TREATMENT| |a respiratory tract infection |PROBLEM | |metformin |TREATMENT| |glipizide |TREATMENT| |dapagliflozin |TREATMENT| |T2DM |PROBLEM | |atorvastatin |TREATMENT| |gemfibrozil |TREATMENT| +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_clinical| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentense length:|128| ## Data Source Trained on augmented 2010 i2b2 challenge data with ‘embeddings_clinical’. https://portal.dbmi.hms.harvard.edu/projects/n2c2-nlp/ ## Benchmarking ```bash label precision recall f1-score support PROBLEM 0.88 0.92 0.90 30276 TEST 0.91 0.86 0.88 17237 TREATMENT 0.87 0.88 0.88 17298 O 0.97 0.97 0.97 202438 accuracy - - 0.95 267249 macro-avg 0.91 0.91 0.91 267249 weighted-avg 0.95 0.95 0.95 267249 ``` --- layout: model title: Indonesian BertForQuestionAnswering Base Cased model (from cahya) author: John Snow Labs name: bert_qa_base_indonesian_tydiqa date: 2022-07-07 tags: [id, open_source, bert, question_answering] task: Question Answering language: id edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-indonesian-tydiqa` is a Indonesian model originally trained by `cahya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_indonesian_tydiqa_id_4.0.0_3.0_1657183004627.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_indonesian_tydiqa_id_4.0.0_3.0_1657183004627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_indonesian_tydiqa","id") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Siapa namaku?", "Nama saya Clara dan saya tinggal di Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_indonesian_tydiqa","id") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Siapa namaku?", "Nama saya Clara dan saya tinggal di Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_indonesian_tydiqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|id| |Size:|413.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/cahya/bert-base-indonesian-tydiqa --- layout: model title: English RobertaForQuestionAnswering Cased model (from vuiseng9) author: John Snow Labs name: roberta_qa_l_squadv1.1 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-l-squadv1.1` is a English model originally trained by `vuiseng9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_l_squadv1.1_en_4.3.0_3.0_1674220913524.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_l_squadv1.1_en_4.3.0_3.0_1674220913524.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_l_squadv1.1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_l_squadv1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_l_squadv1.1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/vuiseng9/roberta-l-squadv1.1 --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa TFWav2Vec2ForCTC from masapasa author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa` is a English model originally trained by masapasa. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa_en_4.2.0_3.0_1664096613190.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa_en_4.2.0_3.0_1664096613190.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_masapasa| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|756.6 MB| --- layout: model title: German Bert Embeddings(Uncased) author: John Snow Labs name: bert_embeddings_bert_base_german_dbmdz_uncased date: 2022-04-11 tags: [bert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-german-dbmdz-uncased` is a German model orginally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_dbmdz_uncased_de_3.4.2_3.0_1649675809759.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_german_dbmdz_uncased_de_3.4.2_3.0_1649675809759.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_german_dbmdz_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/bert-base-german-dbmdz-uncased --- layout: model title: English asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8 TFWav2Vec2ForCTC from lilitket author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8` is a English model originally trained by lilitket. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8_en_4.2.0_3.0_1664120453852.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8_en_4.2.0_3.0_1664120453852.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr8| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Japanese Part of Speech Tagger (from KoichiYasuoka) author: John Snow Labs name: bert_pos_bert_large_japanese_luw_upos date: 2022-05-09 tags: [bert, pos, part_of_speech, ja, open_source] task: Part of Speech Tagging language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-japanese-luw-upos` is a Japanese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_japanese_luw_upos_ja_3.4.2_3.0_1652092112309.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_large_japanese_luw_upos_ja_3.4.2_3.0_1652092112309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_large_japanese_luw_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ja| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/bert-large-japanese-luw-upos - https://universaldependencies.org/u/pos/ - http://id.nii.ac.jp/1001/00216223/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Extract Entities in Spanish Clinical Trial Abstracts author: John Snow Labs name: ner_clinical_trials_abstracts date: 2022-08-12 tags: [es, clinical, licensed, ner, clinical_abstracts, chem, diso, proc] task: Named Entity Recognition language: es edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Named Entity Recognition model detects relevant entities from Spanish clinical trial abstracts and trained by using MedicalNerApproach annotator that allows to train generic NER models based on Neural Networks. The model detects CHEM (Pharmacological and Chemical Substances), pathologies (DISO), and lab tests, diagnostic or therapeutic procedures (PROC). ## Predicted Entities `CHEM`, `DISO`, `PROC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_trials_abstracts_es_4.0.2_3.0_1660339167613.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_trials_abstracts_es_4.0.2_3.0_1660339167613.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_clinical_trials_abstracts', "es", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols("sentence","token") .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_clinical_trials_abstracts", "es", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.clinical_trial_abstracts").predict("""Efecto de la suplementación con ácido fólico sobre los niveles de homocisteína total en pacientes en hemodiálisis. La hiperhomocisteinemia es un marcador de riesgo independiente de morbimortalidad cardiovascular. Hemos prospectivamente reducir los niveles de homocisteína total (tHcy) mediante suplemento con ácido fólico y vitamina B6 (pp), valorando su posible correlación con dosis de diálisis, función residual y parámetros nutricionales.""") ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |suplementación |PROC | |ácido fólico |CHEM | |niveles de homocisteína |PROC | |hemodiálisis |PROC | |hiperhomocisteinemia |DISO | |niveles de homocisteína total|PROC | |tHcy |PROC | |ácido fólico |CHEM | |vitamina B6 |CHEM | |pp |CHEM | |diálisis |PROC | |función residual |PROC | +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_trials_abstracts| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|16.4 MB| ## References The model is prepared using the reference paper: "A clinical trials corpus annotated with UMLS entities to enhance the access to evidence-based medicine", Leonardo Campillos-Llanos, Ana Valverde-Mateos, Adrián Capllonch-Carrión and Antonio Moreno-Sandoval. BMC Medical Informatics and Decision Making volume 21, Article number: 69 (2021) ## Benchmarking ```bash label precision recall f1-score support B-DISO 0.91 0.93 0.92 2465 I-DISO 0.85 0.88 0.86 2788 B-CHEM 0.91 0.92 0.91 1558 I-CHEM 0.82 0.91 0.86 645 B-PROC 0.89 0.91 0.90 3348 I-PROC 0.80 0.87 0.83 4232 micro-avg 0.86 0.90 0.88 15036 macro-avg 0.86 0.90 0.88 15036 weighted-avg 0.86 0.90 0.88 15036 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from clevrly) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_hotpot date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-hotpot_qa` is a English model originally trained by `clevrly`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_hotpot_en_4.3.0_3.0_1672767990673.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_hotpot_en_4.3.0_3.0_1672767990673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_hotpot","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_hotpot","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_hotpot| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/clevrly/distilbert-base-uncased-finetuned-hotpot_qa --- layout: model title: Pipeline to Detect Chemical Compounds and Genes (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_chemprot_pipeline date: 2023-03-20 tags: [berfortokenclassification, chemprot, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemprot](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemprot_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_4.3.0_3.2_1679306959462.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_4.3.0_3.2_1679306959462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") text = '''Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") val text = "Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemprot_pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:------------|-------------:| | 0 | Keratinocyte | 0 | 11 | GENE-Y | 0.999147 | | 1 | growth | 13 | 18 | GENE-Y | 0.999752 | | 2 | factor | 20 | 25 | GENE-Y | 0.999685 | | 3 | acidic | 31 | 36 | GENE-Y | 0.999661 | | 4 | fibroblast | 38 | 47 | GENE-Y | 0.999753 | | 5 | growth | 49 | 54 | GENE-Y | 0.999771 | | 6 | factor | 56 | 61 | GENE-Y | 0.999742 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemprot_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English Bert Embeddings (Base, Uncased, Unstructured, Without Classifier Layer) author: John Snow Labs name: bert_embeddings_bert_base_uncased_mnli_sparse_70_unstructured_no_classifier date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-uncased-mnli-sparse-70-unstructured-no-classifier` is a English model orginally trained by `Intel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_uncased_mnli_sparse_70_unstructured_no_classifier_en_3.4.2_3.0_1649672994004.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_uncased_mnli_sparse_70_unstructured_no_classifier_en_3.4.2_3.0_1649672994004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_uncased_mnli_sparse_70_unstructured_no_classifier| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|228.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/Intel/bert-base-uncased-mnli-sparse-70-unstructured-no-classifier --- layout: model title: English T5ForConditionalGeneration Cased model (from pitehu) author: John Snow Labs name: t5_ner_conll_list date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `T5_NER_CONLL_LIST` is a English model originally trained by `pitehu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_ner_conll_list_en_4.3.0_3.0_1675099601757.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_ner_conll_list_en_4.3.0_3.0_1675099601757.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ner_conll_list","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ner_conll_list","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_ner_conll_list| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|272.6 MB| ## References - https://huggingface.co/pitehu/T5_NER_CONLL_LIST --- layout: model title: English Named Entity Recognition (from Jorgeutd) author: John Snow Labs name: bert_ner_bert_large_uncased_finetuned_ner date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-uncased-finetuned-ner` is a English model orginally trained by `Jorgeutd`. ## Predicted Entities `LOC`, `PER`, `ORG`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_large_uncased_finetuned_ner_en_3.4.2_3.0_1652097206579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_large_uncased_finetuned_ner_en_3.4.2_3.0_1652097206579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_large_uncased_finetuned_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Jorgeutd/bert-large-uncased-finetuned-ner - https://paperswithcode.com/sota?task=Token+Classification&dataset=conll2003 --- layout: model title: Pipeline for detecting posology entities author: John Snow Labs name: recognize_entities_posology date: 2021-04-01 tags: [pipeline, en, licensed, clinical] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pipeline with `ner_posology`. It will only extract medication entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/recognize_entities_posology_en_3.0.0_3.0_1617298186572.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/recognize_entities_posology_en_3.0.0_3.0_1617298186572.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('recognize_entities_posology', 'en', 'clinical/models') res = pipeline.fullAnnotate("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals. """) ``` ```scala val era_pipeline = new PretrainedPipeline("recognize_entities_posology", "en", "clinical/models") val result = era_pipeline.fullAnnotate("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals. """)(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.recognize_entities.posology").predict("""A 28-year-old female with a history of gestational diabetes mellitus, used to take metformin 1000 mg two times a day, presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting . She was seen by the endocrinology service and discharged on 40 units of insulin glargine at night, 12 units of insulin lispro with meals. """) ```

## Results ```bash | | chunk | begin | end | entity | |---:|:-----------------|--------:|------:|:----------| | 0 | metformin | 83 | 91 | DRUG | | 1 | 1000 mg | 93 | 99 | STRENGTH | | 2 | two times a day | 101 | 115 | FREQUENCY | | 3 | 40 units | 270 | 277 | DOSAGE | | 4 | insulin glargine | 282 | 297 | DRUG | | 5 | at night | 299 | 306 | FREQUENCY | | 6 | 12 units | 309 | 316 | DOSAGE | | 7 | insulin lispro | 321 | 334 | DRUG | | 8 | with meals | 336 | 345 | FREQUENCY | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|recognize_entities_posology| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: English XlmRoBertaForQuestionAnswering (from seongju) author: John Snow Labs name: xlm_roberta_qa_klue_mrc_roberta_base date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `klue-mrc-roberta-base` is a English model originally trained by `seongju`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_klue_mrc_roberta_base_en_4.0.0_3.0_1655987736396.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_klue_mrc_roberta_base_en_4.0.0_3.0_1655987736396.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_klue_mrc_roberta_base","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_klue_mrc_roberta_base","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.klue.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_klue_mrc_roberta_base| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|855.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/seongju/klue-mrc-roberta-base --- layout: model title: English DistilBertForQuestionAnswering model (from akr) author: John Snow Labs name: distilbert_qa_akr_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `akr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_akr_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724968396.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_akr_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724968396.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_akr_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_akr_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_akr").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_akr_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/akr/distilbert-base-uncased-finetuned-squad --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dm1000 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dm1000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm1000_en_4.3.0_3.0_1675110576936.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm1000_en_4.3.0_3.0_1675110576936.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dm1000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dm1000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dm1000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|633.1 MB| ## References - https://huggingface.co/google/t5-efficient-base-dm1000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Pipeline to Summarize Clinical Notes (PubMed) author: John Snow Labs name: summarizer_biomedical_pubmed_pipeline date: 2023-05-31 tags: [licensed, en, clinical, summarization] task: Summarization language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_biomedical_pubmed](https://nlp.johnsnowlabs.com/2023/04/03/summarizer_biomedical_pubmed_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_biomedical_pubmed_pipeline_en_4.4.1_3.0_1685533294249.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_biomedical_pubmed_pipeline_en_4.4.1_3.0_1685533294249.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_biomedical_pubmed_pipeline", "en", "clinical/models") text = """Residual disease after initial surgery for ovarian cancer is the strongest prognostic factor for survival. However, the extent of surgical resection required to achieve optimal cytoreduction is controversial. Our goal was to estimate the effect of aggressive surgical resection on ovarian cancer patient survival.\n A retrospective cohort study of consecutive patients with International Federation of Gynecology and Obstetrics stage IIIC ovarian cancer undergoing primary surgery was conducted between January 1, 1994, and December 31, 1998. The main outcome measures were residual disease after cytoreduction, frequency of radical surgical resection, and 5-year disease-specific survival.\n The study comprised 194 patients, including 144 with carcinomatosis. The mean patient age and follow-up time were 64.4 and 3.5 years, respectively. After surgery, 131 (67.5%) of the 194 patients had less than 1 cm of residual disease (definition of optimal cytoreduction). Considering all patients, residual disease was the only independent predictor of survival; the need to perform radical procedures to achieve optimal cytoreduction was not associated with a decrease in survival. For the subgroup of patients with carcinomatosis, residual disease and the performance of radical surgical procedures were the only independent predictors. Disease-specific survival was markedly improved for patients with carcinomatosis operated on by surgeons who most frequently used radical procedures compared with those least likely to use radical procedures (44% versus 17%, P < .001).\n Overall, residual disease was the only independent predictor of survival. Minimizing residual disease through aggressive surgical resection was beneficial, especially in patients with carcinomatosis.""" result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_biomedical_pubmed_pipeline", "en", "clinical/models") val text = """Residual disease after initial surgery for ovarian cancer is the strongest prognostic factor for survival. However, the extent of surgical resection required to achieve optimal cytoreduction is controversial. Our goal was to estimate the effect of aggressive surgical resection on ovarian cancer patient survival.\n A retrospective cohort study of consecutive patients with International Federation of Gynecology and Obstetrics stage IIIC ovarian cancer undergoing primary surgery was conducted between January 1, 1994, and December 31, 1998. The main outcome measures were residual disease after cytoreduction, frequency of radical surgical resection, and 5-year disease-specific survival.\n The study comprised 194 patients, including 144 with carcinomatosis. The mean patient age and follow-up time were 64.4 and 3.5 years, respectively. After surgery, 131 (67.5%) of the 194 patients had less than 1 cm of residual disease (definition of optimal cytoreduction). Considering all patients, residual disease was the only independent predictor of survival; the need to perform radical procedures to achieve optimal cytoreduction was not associated with a decrease in survival. For the subgroup of patients with carcinomatosis, residual disease and the performance of radical surgical procedures were the only independent predictors. Disease-specific survival was markedly improved for patients with carcinomatosis operated on by surgeons who most frequently used radical procedures compared with those least likely to use radical procedures (44% versus 17%, P < .001).\n Overall, residual disease was the only independent predictor of survival. Minimizing residual disease through aggressive surgical resection was beneficial, especially in patients with carcinomatosis.""" val result = pipeline.fullAnnotate(text) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_uncased_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_uncased_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|307.1 MB| |Case sensitive:|false| |Max sentence length:|256| ## References - https://huggingface.co/t15/distilroberta-base-uncased-squad --- layout: model title: English RobertaForTokenClassification Cased model (from Jean-Baptiste) author: John Snow Labs name: roberta_token_classifier_ticker date: 2023-03-01 tags: [en, open_source, roberta, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-ticker` is a English model originally trained by `Jean-Baptiste`. ## Predicted Entities `TICKER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_ticker_en_4.3.0_3.0_1677703811345.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_ticker_en_4.3.0_3.0_1677703811345.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_ticker","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_ticker","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_ticker| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|465.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Jean-Baptiste/roberta-ticker - https://www.kaggle.com/omermetinn/tweets-about-the-top-companies-from-2015-to-2020 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab40 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab40 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab40` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab40_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab40_en_4.2.0_3.0_1664020884314.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab40_en_4.2.0_3.0_1664020884314.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab40', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab40", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab40| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Detect Symptoms, Treatments and Other Entities in German author: John Snow Labs name: ner_healthcare date: 2021-03-31 tags: [ner, clinical, licensed, de] task: Named Entity Recognition language: de edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect symptoms, treatments and other entities in medical text in German language. ## Predicted Entities `DIAGLAB_PROCEDURE`, `MEDICAL_SPECIFICATION`, `MEDICAL_DEVICE`, `MEASUREMENT`, `BIOLOGICAL_CHEMISTRY`, `BODY_FLUID`, `TIME_INFORMATION`, `LOCAL_SPECIFICATION`, `BIOLOGICAL_PARAMETER`, `PROCESS`, `MEDICATION`, `DOSING`, `DEGREE`, `MEDICAL_CONDITION`, `PERSON`, `TISSUE`, `STATE_OF_HEALTH`, `BODY_PART`, `TREATMENT` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HEALTHCARE_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_de_3.0.0_3.0_1617208455368.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_de_3.0.0_3.0_1617208455368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------+---------------------+-----+---+ |chunk |ner_label |begin|end| +-----------------+---------------------+-----+---+ |Kleinzellige |MEASUREMENT |4 |15 | |Bronchialkarzinom|MEDICAL_CONDITION |17 |33 | |Kleinzelliger |MEDICAL_SPECIFICATION|36 |48 | |Lungenkrebs |MEDICAL_CONDITION |50 |60 | |SCLC |MEDICAL_CONDITION |63 |66 | |Karzinom |MEDICAL_CONDITION |103 |110| +-----------------+---------------------+-----+---+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with *w2v_cc_300d*. ## Benchmarking ```bash | | label | tp | fp | fn | precision| recall| f1 | |---:|--------------------:|-------:|------:|-----:|---------:|---------:|---------:| | 0 | BIOLOGICAL_PARAMETER| 103 | 52 | 57 | 0.6645 | 0.6438 | 0.654 | | 1 | BODY_FLUID | 166 | 16 | 24 | 0.9121 | 0.8737 | 0.8925 | | 2 | PERSON | 475 | 74 | 142 | 0.8652 | 0.7699 | 0.8148 | | 3 | DOSING | 38 | 14 | 31 | 0.7308 | 0.5507 | 0.6281 | | 4 | DIAGLAB_PROCEDURE | 236 | 58 | 68 | 0.8027 | 0.7763 | 0.7893 | | 5 | BODY_PART | 690 | 72 | 79 | 0.9055 | 0.8973 | 0.9014 | | 6 | MEDICATION | 391 | 117 | 167 | 0.7697 | 0.7007 | 0.7336 | | 7 | STATE_OF_HEALTH | 321 | 41 | 76 | 0.8867 | 0.8086 | 0.8458 | | 8 | LOCAL_SPECIFICATION | 57 | 19 | 24 | 0.75 | 0.7037 | 0.7261 | | 9 | MEASUREMENT | 574 | 260 | 222 | 0.6882 | 0.7211 | 0.7043 | | 10 | TREATMENT | 476 | 131 | 135 | 0.7842 | 0.7791 | 0.7816 | | 11 | MEDICAL_CONDITION | 1741 | 442 | 271 | 0.7975 | 0.8653 | 0.83 | | 12 | TIME_INFORMATION | 651 | 126 | 161 | 0.8378 | 0.8017 | 0.8194 | | 13 | BIOLOGICAL_CHEMISTRY| 192 | 55 | 60 | 0.7773 | 0.7619 | 0.7695 | ``` --- layout: model title: Chuvash RobertaForQuestionAnswering (from sunitha) author: John Snow Labs name: roberta_qa_CV_Merge_DS date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: cv edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `CV_Merge_DS` is a Chuvash model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_CV_Merge_DS_cv_4.0.0_3.0_1655726646554.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_CV_Merge_DS_cv_4.0.0_3.0_1655726646554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_CV_Merge_DS","cv") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_CV_Merge_DS","cv") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("cv.answer_question.roberta.cv_merge_ds.by_sunitha").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_CV_Merge_DS| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|cv| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/CV_Merge_DS --- layout: model title: Legal Master Lease Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_master_lease_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, master_lease, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_master_lease_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `master-lease-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `master-lease-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_master_lease_agreement_en_1.0.0_3.0_1668117131869.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_master_lease_agreement_en_1.0.0_3.0_1668117131869.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[master-lease-agreement]| |[other]| |[other]| |[master-lease-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_master_lease_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support master-lease-agreement 0.96 0.90 0.93 30 other 0.96 0.98 0.97 66 accuracy - - 0.96 96 macro-avg 0.96 0.94 0.95 96 weighted-avg 0.96 0.96 0.96 96 ``` --- layout: model title: Swedish BertForQuestionAnswering model (from susumu2357) author: John Snow Labs name: bert_qa_bert_base_swedish_squad2 date: 2022-06-02 tags: [sv, open_source, question_answering, bert] task: Question Answering language: sv edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-swedish-squad2` is a Swedish model orginally trained by `susumu2357`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_swedish_squad2_sv_4.0.0_3.0_1654180667162.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_swedish_squad2_sv_4.0.0_3.0_1654180667162.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_swedish_squad2","sv") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_swedish_squad2","sv") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("sv.answer_question.squadv2.bert.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_swedish_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|sv| |Size:|465.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/susumu2357/bert-base-swedish-squad2 - https://github.com/susumu2357/SQuAD_v2_sv --- layout: model title: Legal Capitalization Clause Binary Classifier author: John Snow Labs name: legclf_capitalization_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `capitalization` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `capitalization` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_capitalization_clause_en_1.0.0_3.2_1660122203093.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_capitalization_clause_en_1.0.0_3.2_1660122203093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_capitalization_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[capitalization]| |[other]| |[other]| |[capitalization]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_capitalization_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support capitalization 1.00 0.89 0.94 19 other 0.96 1.00 0.98 53 accuracy - - 0.97 72 macro-avg 0.98 0.95 0.96 72 weighted-avg 0.97 0.97 0.97 72 ``` --- layout: model title: English XlmRoBertaForQuestionAnswering (from meghana) author: John Snow Labs name: xlm_roberta_qa_hitalmqa_finetuned_squad date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `hitalmqa-finetuned-squad` is a English model originally trained by `meghana`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_hitalmqa_finetuned_squad_en_4.0.0_3.0_1655987596935.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_hitalmqa_finetuned_squad_en_4.0.0_3.0_1655987596935.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_hitalmqa_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_hitalmqa_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.xlm_roberta.by_meghana").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_hitalmqa_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/meghana/hitalmqa-finetuned-squad --- layout: model title: Clinical Deidentification author: John Snow Labs name: clinical_deidentification class: PipelineModel language: en nav_key: models repository: clinical/models date: 2020-01-31 task: [De-identification, Pipeline Healthcare] edition: Healthcare NLP 2.4.0 spark_version: 2.4 tags: [pipeline, clinical, licensed] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This pipeline can be used to de-identify PHI information from medical texts. The PHI information will be obfuscated in the resulting text. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_2.4.0_2.4_1580481115376.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_2.4.0_2.4_1580481115376.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" model = PretrainedPipeline("clinical_deidentification", "en", "clinical/models") result = deid_pipeline.annotate(sample) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" val model = new PretrainedPipeline("clinical_deidentification", "en", "clinical/models") val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.clinical_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

{:.model-param} ## Model Information {:.table-model} |---------------|---------------------------| | Name: | clinical_deidentification | | Type: | PipelineModel | | Compatibility: | Spark NLP 2.4.0+ | | License: | Licensed | | Edition: | Official | | Language: | en | --- layout: model title: English asr_wav2vec2_base_timit_demo_colab2_by_sherry7144 TFWav2Vec2ForCTC from sherry7144 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab2_by_sherry7144 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab2_by_sherry7144` is a English model originally trained by sherry7144. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab2_by_sherry7144_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_sherry7144_en_4.2.0_3.0_1664025065732.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_sherry7144_en_4.2.0_3.0_1664025065732.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_sherry7144", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_sherry7144", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab2_by_sherry7144| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Earning Calls Financial NER (Specific, sm) author: John Snow Labs name: finner_earning_calls_specific_sm date: 2022-11-30 tags: [en, finance, ner, licensed, earning, calls] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a `sm` (small) version of a financial model trained on Earning Calls transcripts to detect financial entities (NER model). This model is called `Specific` as it has more labels in comparison with a `Generic` version. Please note this model requires some tokenization configuration to extract the currency (see python snippet below). The currently available entities are: - AMOUNT: Numeric amounts, not percentages - ASSET: Current or Fixed Asset - ASSET_DECREASE: Decrease in the asset possession/exposure - ASSET_INCREASE: Increase in the asset possession/exposure - CF: Total cash flow - CFO: Cash flow from operating activity - CFO_INCREASE: Cash flow from operating activity increased - COUNT: Number of items (not monetary, not percentages). - CURRENCY: The currency of the amount - DATE: Generic dates in context where either it's not a fiscal year or it can't be asserted as such given the context - EXPENSE: An expense or loss - EXPENSE_DECREASE: A piece of information saying there was an expense decrease in that fiscal year - EXPENSE_INCREASE: A piece of information saying there was an expense increase in that fiscal year - FCF: Free Cash Flow - FISCAL_YEAR: A date which expresses which month the fiscal exercise was closed for a specific year - INCOME: Any income that is reported - INCOME_INCREASE: Relative increase in income - KPI: Key Performance Indicator, a quantifiable measure of performance over time for a specific objective - KPI_DECREASE: Relative decrease in a KPI - KPI_INCREASE: Relative increase in a KPI - LIABILITY: Current or Long-Term Liability (not from stockholders) - LIABILITY_DECREASE: Relative decrease in liability - LIABILITY_INCREASE: Relative increase in liability - LOSS: Type of loss (e.g. gross, net) - ORG: Mention to a company/organization name - PERCENTAGE: : Numeric amounts which are percentages - PROFIT: Profit or also Revenue - PROFIT_DECLINE: A piece of information saying there was a profit / revenue decrease in that fiscal year - PROFIT_INCREASE: A piece of information saying there was a profit / revenue increase in that fiscal year - REVENUE: Revenue reported by company - REVENUE_DECLINE: Relative decrease in revenue when compared to other years - REVENUE_INCREASE: Relative increase in revenue when compared to other years - STOCKHOLDERS_EQUITY: Equity possessed by stockholders, not liability - TICKER: Trading symbol of the company ## Predicted Entities `AMOUNT`, `ASSET`, `ASSET_DECREASE`, `ASSET_INCREASE`, `CF`, `CFO`, `CFO_INCREASE`, `COUNT`, `CURRENCY`, `DATE`, `EXPENSE`, `EXPENSE_DECREASE`, `EXPENSE_INCREASE`, `FCF`, `FISCAL_YEAR`, `INCOME`, `INCOME_INCREASE`, `KPI`, `KPI_DECREASE`, `KPI_INCREASE`, `LIABILITY`, `LIABILITY_DECREASE`, `LIABILITY_INCREASE`, `LOSS`, `ORG`, `PERCENTAGE`, `PROFIT`, `PROFIT_DECLINE`, `PROFIT_INCREASE`, `REVENUE`, `REVENUE_DECLINE`, `REVENUE_INCREASE`, `STOCKHOLDERS_EQUITY`, `TICKER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_specific_sm_en_1.0.0_3.0_1669835090214.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_specific_sm_en_1.0.0_3.0_1669835090214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_earning_calls_specific_sm", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Adjusted EPS was ahead of our expectations at $ 1.21 , and free cash flow is also ahead of our expectations despite a $ 1.5 billion additional tax payment we made related to the R&D amortization."""]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("text"), F.expr("cols['1']['entity']").alias("label")).show(200, truncate = False) ```

## Results ```bash +------------+----------+----------+ | token| ner_label|confidence| +------------+----------+----------+ | Adjusted| B-PROFIT| 0.6957| | EPS| I-PROFIT| 0.8325| | was| O| 0.9994| | ahead| O| 0.9996| | of| O| 0.9929| | our| O| 0.9852| |expectations| O| 0.9845| | at| O| 1.0| | $|B-CURRENCY| 0.9995| | 1.21| B-AMOUNT| 1.0| | ,| O| 0.9993| | and| O| 0.9997| | free| B-FCF| 0.9883| | cash| I-FCF| 0.815| | flow| I-FCF| 0.8644| | is| O| 0.9997| | also| O| 0.9966| | ahead| O| 0.9998| | of| O| 0.9953| | our| O| 0.9877| |expectations| O| 0.994| | despite| O| 0.9997| | a| O| 0.9979| | $|B-CURRENCY| 0.9992| | 1.5| B-AMOUNT| 1.0| | billion| I-AMOUNT| 0.9997| | additional| B-EXPENSE| 0.641| | tax| I-EXPENSE| 0.3146| | payment| I-EXPENSE| 0.6099| | we| O| 0.9613| | made| O| 0.982| | related| O| 0.9732| | to| O| 0.8816| | the| O| 0.7283| | R&D| O| 0.8978| |amortization| O| 0.5825| | .| O| 1.0| +------------+----------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_earning_calls_specific_sm| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on Earning Calls. ## Benchmarking ```bash label tp fp fn prec rec f1 I-REVENUE_INCREASE 53 16 52 0.76811594 0.50476193 0.6091954 I-AMOUNT 382 2 4 0.9947917 0.9896373 0.9922078 B-COUNT 14 9 1 0.6086956 0.93333334 0.73684216 B-AMOUNT 454 0 5 1.0 0.9891068 0.9945236 I-KPI 2 11 23 0.15384616 0.08 0.10526316 I-ORG 16 0 0 1.0 1.0 1.0 B-DATE 122 13 0 0.9037037 1.0 0.94941634 B-LIABILITY_DECREASE 1 1 0 0.5 1.0 0.6666667 I-DATE 3 2 0 0.6 1.0 0.75 B-LOSS 4 0 4 1.0 0.5 0.6666667 I-ASSET 6 2 14 0.75 0.3 0.42857146 I-EXPENSE 46 13 59 0.77966 0.43809524 0.5609756 I-KPI_INCREASE 1 7 13 0.125 0.071428575 0.09090909 B-REVENUE_INCREASE 60 21 34 0.7407407 0.63829786 0.6857143 I-COUNT 13 6 0 0.68421054 1.0 0.8125 I-CFO 23 1 0 0.9583333 1.0 0.9787234 B-FCF 13 4 0 0.7647059 1.0 0.8666667 B-PROFIT_INCREASE 11 11 5 0.5 0.6875 0.57894737 B-EXPENSE 26 16 45 0.61904764 0.36619717 0.460177 B-REVENUE_DECLINE 6 4 13 0.6 0.31578946 0.41379312 B-STOCKHOLDERS_EQUITY 3 0 3 1.0 0.5 0.6666667 I-PROFIT_DECLINE 4 1 7 0.8 0.36363637 0.5 I-LIABILITY_DECREASE 1 1 0 0.5 1.0 0.6666667 I-LOSS 12 0 10 1.0 0.54545456 0.7058824 I-PROFIT 148 40 10 0.78723407 0.93670887 0.8554913 B-CFO 9 1 1 0.9 0.9 0.9 B-CURRENCY 440 0 1 1.0 0.9977324 0.9988649 I-PROFIT_INCREASE 11 10 6 0.52380955 0.64705884 0.5789474 I-CURRENCY 6 0 0 1.0 1.0 1.0 B-PROFIT 93 27 16 0.775 0.853211 0.812227 B-PERCENTAGE 418 7 3 0.9835294 0.9928741 0.9881796 B-TICKER 13 0 0 1.0 1.0 1.0 I-FISCAL_YEAR 2 3 1 0.4 0.6666667 0.5 B-ORG 14 0 0 1.0 1.0 1.0 I-STOCKHOLDERS_EQUITY 6 0 2 1.0 0.75 0.85714287 I-REVENUE_DECLINE 8 9 8 0.47058824 0.5 0.4848485 B-EXPENSE_INCREASE 6 0 4 1.0 0.6 0.75 B-REVENUE 51 17 15 0.75 0.77272725 0.761194 B-FISCAL_YEAR 1 1 0 0.5 1.0 0.6666667 I-EXPENSE_DECREASE 3 3 2 0.5 0.6 0.54545456 I-FCF 26 9 0 0.74285716 1.0 0.852459 I-REVENUE 45 12 18 0.7894737 0.71428573 0.75000006 I-EXPENSE_INCREASE 8 0 3 1.0 0.72727275 0.84210527 Macro-average 2611 311 491 0.6658762 0.6029909 0.63287526 Micro-average 2611 311 91 0.8935661 0.84171504 0.8668659 ``` --- layout: model title: Pipeline to Extract Pharmacological Entities from Spanish Medical Texts author: John Snow Labs name: ner_pharmacology_pipeline date: 2023-03-09 tags: [es, clinical, licensed, ner, pharmacology, proteinas, normalizables] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_pharmacology](https://nlp.johnsnowlabs.com/2022/08/13/ner_pharmacology_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_pharmacology_pipeline_es_4.3.0_3.2_1678358547733.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_pharmacology_pipeline_es_4.3.0_3.2_1678358547733.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_pharmacology_pipeline", "es", "clinical/models") text = '''e realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_pharmacology_pipeline", "es", "clinical/models") val text = "e realiza analítica destacando creatinkinasa 736 UI, LDH 545 UI, urea 63 mg/dl, CA 19.9 64,1 U/ml. Inmunofenotípicamente el tumor expresó vimentina, S-100, HMB-45 y actina. Se instauró el tratamiento con quimioterapia (Cisplatino, Interleukina II, Dacarbacina e Interferon alfa)." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------|--------:|------:|:--------------|-------------:| | 0 | creatinkinasa | 31 | 43 | PROTEINAS | 0.9994 | | 1 | LDH | 53 | 55 | PROTEINAS | 0.9996 | | 2 | urea | 65 | 68 | NORMALIZABLES | 0.9996 | | 3 | CA 19.9 | 80 | 86 | PROTEINAS | 0.99835 | | 4 | vimentina | 138 | 146 | PROTEINAS | 0.9991 | | 5 | S-100 | 149 | 153 | PROTEINAS | 0.9996 | | 6 | HMB-45 | 156 | 161 | PROTEINAS | 0.9986 | | 7 | actina | 165 | 170 | PROTEINAS | 0.9998 | | 8 | Cisplatino | 219 | 228 | NORMALIZABLES | 0.9999 | | 9 | Interleukina II | 231 | 245 | PROTEINAS | 0.99955 | | 10 | Dacarbacina | 248 | 258 | NORMALIZABLES | 0.9996 | | 11 | Interferon alfa | 262 | 276 | PROTEINAS | 0.99935 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_pharmacology_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|318.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - RoBertaEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Greek BertForQuestionAnswering Cased model (from Danastos) author: John Snow Labs name: bert_qa_nq_el_4 date: 2022-07-07 tags: [el, open_source, bert, question_answering] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `nq_bert_el_4` is a Greek model originally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_nq_el_4_el_4.0.0_3.0_1657190609566.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_nq_el_4_el_4.0.0_3.0_1657190609566.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_nq_el_4","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_nq_el_4","el") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_nq_el_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|el| |Size:|421.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/nq_bert_el_4 --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_by_oliverguhr TFWav2Vec2ForCTC from oliverguhr author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_german_by_oliverguhr date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_by_oliverguhr` is a German model originally trained by oliverguhr. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_de_4.2.0_3.0_1664104475873.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_by_oliverguhr_de_4.2.0_3.0_1664104475873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_german_by_oliverguhr", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_german_by_oliverguhr", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_german_by_oliverguhr| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Russian BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_ru_cased date: 2022-12-02 tags: [ru, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ru edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-ru-cased` is a Russian model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_ru_cased_ru_4.2.4_3.0_1670018825264.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_ru_cased_ru_4.2.4_3.0_1670018825264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_ru_cased","ru") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_ru_cased","ru") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_ru_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ru| |Size:|364.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-ru-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Spanish NER Model author: John Snow Labs name: roberta_token_classifier_bne_capitel_ner date: 2021-12-07 tags: [roberta, spanish, es, token_classifier, open_source] task: Named Entity Recognition language: es edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description - This model is imported from `Hugging Face`. - RoBERTa-base-bne is a transformer-based masked language model for the Spanish language. It is based on the RoBERTa base model and has been pretrained using the largest Spanish corpus known to date, with a total of 570GB of clean and deduplicated text processed for this work, compiled from the web crawlings performed by the National Library of Spain (Biblioteca Nacional de España) from 2009 to 2019. ## Predicted Entities `OTH`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bne_capitel_ner_es_3.3.2_2.4_1638866935540.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bne_capitel_ner_es_3.3.2_2.4_1638866935540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------+---------+ |chunk |ner_label| +------------------------+---------+ |Antonio |PER | |fábrica de Mercedes-Benz|ORG | |Madrid. |LOC | +------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_bne_capitel_ner| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|es| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne-capitel-ner-plus](https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne-capitel-ner-plus) ## Benchmarking ```bash label score f1 0.8867 ``` --- layout: model title: Detect Genomic Variant Information (ner_genetic_variants) author: John Snow Labs name: ner_genetic_variants date: 2021-06-25 tags: [ner, en, clinical, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts genetic variant information from the medical text and is trained on the tmVar 2.0 dataset that can be found at https://www.ncbi.nlm.nih.gov/research/bionlp/Tools/tmvar/ ## Predicted Entities `DNAMutation`, `ProteinMutation`, `SNP` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_GEN_VAR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_genetic_variants_en_3.1.0_3.0_1624607526370.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_genetic_variants_en_3.1.0_3.0_1624607526370.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained('embeddings_clinical', 'en', 'clinical/models') \ .setInputCols(['sentence', 'token']) \ .setOutputCol('embeddings') clinical_ner = MedicalNerModel.pretrained("ner_genetic_variants", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) data = spark.createDataFrame([["""The mutation pattern of mitochondrial DNA (mtDNA) in mainland Chinese patients with mitochondrial myopathy, encephalopathy, lactic acidosis and stroke-like episodes (MELAS) has been rarely reported, though previous data suggested that the mutation pattern of MELAS could be different among geographically localized populations. We presented the results of comprehensive mtDNA mutation analysis in 92 unrelated Chinese patients with MELAS (85 with classic MELAS and 7 with MELAS/Leigh syndrome (LS) overlap syndrome). The mtDNA A3243G mutation was the most common causal genotype in this patient group (79/92 and 85.9%). The second common gene mutation was G13513A (7/92 and 7.6%). Additionally, we identified T10191C (p.S45P) in ND3, A11470C (p. K237N) in ND4, T13046C (p.M237T) in ND5 and a large-scale deletion (13025-13033:14417-14425) involving partial ND5 and ND6 subunits of complex I in one patient each. Among them, A11470C, T13046C and the single deletion were novel mutations. In summary, patients with mutations affecting mitochondrially encoded complex I (MTND) reached 12.0% (11/92) in this group. It is noteworthy that all seven patients with MELAS/LS overlap syndrome were associated with MTND mutations. Our data emphasize the important role of MTND mutations in the pathogenicity of MELAS, especially MELAS/LS overlap syndrome.PURPOSE: Genes in the complement pathway, including complement factor H (CFH), C2/BF, and C3, have been reported to be associated with age-related macular degeneration (AMD). Genetic variants, single-nucleotide polymorphisms (SNPs), in these genes were geno-typed for a case-control association study in a mainland Han Chinese population. METHODS: One hundred and fifty-eight patients with wet AMD, 80 patients with soft drusen, and 220 matched control subjects were recruited among Han Chinese in mainland China. Seven SNPs in CFH and two SNPs in C2, CFB', and C3 were genotyped using the ABI SNaPshot method. A deletion of 84,682 base pairs covering the CFHR1 and CFHR3 genes was detected by direct polymerase chain reaction and gel electrophoresis. RESULTS: Four SNPs, including rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), in CFH showed a significant association with wet AMD in the cohort of this study. A haplotype containing these four SNPs (CATA) significantly increased protection of wet AMD with a P value of 0.0005 and an odds ratio of 0.29 (95% confidence interval: 0.15-0.60). Unlike in other populations, rs2274700 and rs1410996 did not show a significant association with AMD in the Chinese population of this study. None of the SNPs in CFH showed a significant association with drusen, and none of the SNPs in CFH, C2, CFB, and C3 showed a significant association with either wet AMD or drusen in the cohort of this study. The CFHR1 and CFHR3 deletion was not polymorphic in the Chinese population and was not associated with wet AMD or drusen. CONCLUSION: This study showed that SNPs rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), but not rs7535263, rs1410996, or rs2274700, in CFH were significantly associated with wet AMD in a mainland Han Chinese population. This study showed that CFH was more likely to be AMD susceptibility gene at Chr.1q31 based on the finding that the CFHR1 and CFHR3 deletion was not polymorphic in the cohort of this study, and none of the SNPs that were significantly associated with AMD in a white population in C2, CFB, and C3 genes showed a significant association with AMD."""]]).toDF("text") result = nlpPipeline.fit(data).transform(data) ``` ```scala ... val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols("sentence", "token") .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_genetic_variants", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val data = Seq("The mutation pattern of mitochondrial DNA (mtDNA) in mainland Chinese patients with mitochondrial myopathy, encephalopathy, lactic acidosis and stroke-like episodes (MELAS) has been rarely reported, though previous data suggested that the mutation pattern of MELAS could be different among geographically localized populations. We presented the results of comprehensive mtDNA mutation analysis in 92 unrelated Chinese patients with MELAS (85 with classic MELAS and 7 with MELAS/Leigh syndrome (LS) overlap syndrome). The mtDNA A3243G mutation was the most common causal genotype in this patient group (79/92 and 85.9%). The second common gene mutation was G13513A (7/92 and 7.6%). Additionally, we identified T10191C (p.S45P) in ND3, A11470C (p. K237N) in ND4, T13046C (p.M237T) in ND5 and a large-scale deletion (13025-13033:14417-14425) involving partial ND5 and ND6 subunits of complex I in one patient each. Among them, A11470C, T13046C and the single deletion were novel mutations. In summary, patients with mutations affecting mitochondrially encoded complex I (MTND) reached 12.0% (11/92) in this group. It is noteworthy that all seven patients with MELAS/LS overlap syndrome were associated with MTND mutations. Our data emphasize the important role of MTND mutations in the pathogenicity of MELAS, especially MELAS/LS overlap syndrome.PURPOSE: Genes in the complement pathway, including complement factor H (CFH), C2/BF, and C3, have been reported to be associated with age-related macular degeneration (AMD). Genetic variants, single-nucleotide polymorphisms (SNPs), in these genes were geno-typed for a case-control association study in a mainland Han Chinese population. METHODS: One hundred and fifty-eight patients with wet AMD, 80 patients with soft drusen, and 220 matched control subjects were recruited among Han Chinese in mainland China. Seven SNPs in CFH and two SNPs in C2, CFB', and C3 were genotyped using the ABI SNaPshot method. A deletion of 84,682 base pairs covering the CFHR1 and CFHR3 genes was detected by direct polymerase chain reaction and gel electrophoresis. RESULTS: Four SNPs, including rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), in CFH showed a significant association with wet AMD in the cohort of this study. A haplotype containing these four SNPs (CATA) significantly increased protection of wet AMD with a P value of 0.0005 and an odds ratio of 0.29 (95% confidence interval: 0.15-0.60). Unlike in other populations, rs2274700 and rs1410996 did not show a significant association with AMD in the Chinese population of this study. None of the SNPs in CFH showed a significant association with drusen, and none of the SNPs in CFH, C2, CFB, and C3 showed a significant association with either wet AMD or drusen in the cohort of this study. The CFHR1 and CFHR3 deletion was not polymorphic in the Chinese population and was not associated with wet AMD or drusen. CONCLUSION: This study showed that SNPs rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), but not rs7535263, rs1410996, or rs2274700, in CFH were significantly associated with wet AMD in a mainland Han Chinese population. This study showed that CFH was more likely to be AMD susceptibility gene at Chr.1q31 based on the finding that the CFHR1 and CFHR3 deletion was not polymorphic in the cohort of this study, and none of the SNPs that were significantly associated with AMD in a white population in C2, CFB, and C3 genes showed a significant association with AMD.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.genetic_variants").predict("""The mutation pattern of mitochondrial DNA (mtDNA) in mainland Chinese patients with mitochondrial myopathy, encephalopathy, lactic acidosis and stroke-like episodes (MELAS) has been rarely reported, though previous data suggested that the mutation pattern of MELAS could be different among geographically localized populations. We presented the results of comprehensive mtDNA mutation analysis in 92 unrelated Chinese patients with MELAS (85 with classic MELAS and 7 with MELAS/Leigh syndrome (LS) overlap syndrome). The mtDNA A3243G mutation was the most common causal genotype in this patient group (79/92 and 85.9%). The second common gene mutation was G13513A (7/92 and 7.6%). Additionally, we identified T10191C (p.S45P) in ND3, A11470C (p. K237N) in ND4, T13046C (p.M237T) in ND5 and a large-scale deletion (13025-13033:14417-14425) involving partial ND5 and ND6 subunits of complex I in one patient each. Among them, A11470C, T13046C and the single deletion were novel mutations. In summary, patients with mutations affecting mitochondrially encoded complex I (MTND) reached 12.0% (11/92) in this group. It is noteworthy that all seven patients with MELAS/LS overlap syndrome were associated with MTND mutations. Our data emphasize the important role of MTND mutations in the pathogenicity of MELAS, especially MELAS/LS overlap syndrome.PURPOSE: Genes in the complement pathway, including complement factor H (CFH), C2/BF, and C3, have been reported to be associated with age-related macular degeneration (AMD). Genetic variants, single-nucleotide polymorphisms (SNPs), in these genes were geno-typed for a case-control association study in a mainland Han Chinese population. METHODS: One hundred and fifty-eight patients with wet AMD, 80 patients with soft drusen, and 220 matched control subjects were recruited among Han Chinese in mainland China. Seven SNPs in CFH and two SNPs in C2, CFB', and C3 were genotyped using the ABI SNaPshot method. A deletion of 84,682 base pairs covering the CFHR1 and CFHR3 genes was detected by direct polymerase chain reaction and gel electrophoresis. RESULTS: Four SNPs, including rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), in CFH showed a significant association with wet AMD in the cohort of this study. A haplotype containing these four SNPs (CATA) significantly increased protection of wet AMD with a P value of 0.0005 and an odds ratio of 0.29 (95% confidence interval: 0.15-0.60). Unlike in other populations, rs2274700 and rs1410996 did not show a significant association with AMD in the Chinese population of this study. None of the SNPs in CFH showed a significant association with drusen, and none of the SNPs in CFH, C2, CFB, and C3 showed a significant association with either wet AMD or drusen in the cohort of this study. The CFHR1 and CFHR3 deletion was not polymorphic in the Chinese population and was not associated with wet AMD or drusen. CONCLUSION: This study showed that SNPs rs3753394 (P = 0.0276), rs800292 (P = 0.0266), rs1061170 (P = 0.00514), and rs1329428 (P = 0.0089), but not rs7535263, rs1410996, or rs2274700, in CFH were significantly associated with wet AMD in a mainland Han Chinese population. This study showed that CFH was more likely to be AMD susceptibility gene at Chr.1q31 based on the finding that the CFHR1 and CFHR3 deletion was not polymorphic in the cohort of this study, and none of the SNPs that were significantly associated with AMD in a white population in C2, CFB, and C3 genes showed a significant association with AMD.""") ```

## Results ```bash +---------+---------------+ |chunk |ner_label | +---------+---------------+ |A3243G |DNAMutation | |G13513A |DNAMutation | |T10191C |DNAMutation | |p.S45P |ProteinMutation| |A11470C |DNAMutation | |p. K237N |ProteinMutation| |T13046C |DNAMutation | |p. |ProteinMutation| |M237T |ProteinMutation| |A11470C |DNAMutation | |T13046C |DNAMutation | |rs3753394|SNP | |rs800292 |SNP | |rs1061170|SNP | |rs1329428|SNP | |rs2274700|SNP | |rs1410996|SNP | |rs3753394|SNP | |rs800292 |SNP | |rs1061170|SNP | |rs1329428|SNP | |rs7535263|SNP | |rs1410996|SNP | |rs2274700|SNP | +---------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_genetic_variants| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source https://www.ncbi.nlm.nih.gov/research/bionlp/Tools/tmvar/ ## Benchmarking ```bash entity tp fp fn total precision recall f1 SNP 38.0 0.0 0.0 38.0 1.0 1.0 1.0 ProteinMutation 171.0 21.0 19.0 190.0 0.8906 0.9 0.8953 DNAMutation 161.0 47.0 42.0 203.0 0.774 0.7931 0.7835 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from peggyhuang) author: John Snow Labs name: bert_qa_scibert_coqa date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `SciBERT-CoQA` is a English model originally trained by `peggyhuang`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_coqa_en_4.0.0_3.0_1657182460716.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_coqa_en_4.0.0_3.0_1657182460716.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_scibert_coqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_scibert_coqa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_scibert_coqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|410.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/peggyhuang/SciBERT-CoQA --- layout: model title: Legal Modification and waiver Clause Binary Classifier author: John Snow Labs name: legclf_modification_and_waiver_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `modification-and-waiver` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `modification-and-waiver` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_modification_and_waiver_clause_en_1.0.0_3.2_1660122661461.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_modification_and_waiver_clause_en_1.0.0_3.2_1660122661461.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_modification_and_waiver_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[modification-and-waiver]| |[other]| |[other]| |[modification-and-waiver]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_modification_and_waiver_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support modification-and-waiver 0.86 0.88 0.87 42 other 0.95 0.94 0.95 106 accuracy - - 0.93 148 macro-avg 0.91 0.91 0.91 148 weighted-avg 0.93 0.93 0.93 148 ``` --- layout: model title: Multilingual (English, German, Spanish) DistilBertForQuestionAnswering model (from ZYW) Squad author: John Snow Labs name: distilbert_qa_squad_en_de_es_model date: 2022-06-08 tags: [en, de, es, open_source, distilbert, question_answering, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-en-de-es-model` is a English model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_en_de_es_model_xx_4.0.0_3.0_1654728749221.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_en_de_es_model_xx_4.0.0_3.0_1654728749221.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_en_de_es_model","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_en_de_es_model","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.squad.distil_bert.en_de_es_tuned.by_ZYW").predict("""PUT YOUR QUESTION HERE|||"PUT YOUR CONTEXT HERE""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_en_de_es_model| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-en-de-es-model --- layout: model title: Mapping ICD10CM Codes with Corresponding Causes and Claim Analysis Codes author: John Snow Labs name: icd10cm_cause_claim_mapper date: 2023-05-11 tags: [en, licensed, chunk_mapping, icd10cm, cause, claim] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps ICD-10-CM codes, subsequently providing corresponding causes and generating claim analysis codes for each respective ICD-10-CM code. If there is no equivalent claim analysis code, the result will be `None`. ## Predicted Entities `icd10cm_cause`, `icd10cm_claim_analysis_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_cause_claim_mapper_en_4.4.0_3.0_1683819210044.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_cause_claim_mapper_en_4.4.0_3.0_1683819210044.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") chunk_assembler = Doc2Chunk()\ .setInputCols("document")\ .setOutputCol("icd_chunk") chunkerMapper = ChunkMapperModel.pretrained("icd10cm_cause_claim_mapper", "en", "clinical/models")\ .setInputCols(["icd_chunk"])\ .setOutputCol("mappings")\ .setRels(["icd10cm_cause", "icd10cm_claim_analysis_code"]) pipeline = Pipeline().setStages([document_assembler, chunk_assembler, chunkerMapper]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) res = lp.fullAnnotate(["D69.51", "G43.83", "A18.03"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val chunk_assembler = new Doc2Chunk() .setInputCols("document") .setOutputCol("icd_chunk") val chunkerMapper = ChunkMapperModel.pretrained("icd10cm_cause_claim_mapper", "en", "clinical/models") .setInputCols(Array("icd_chunk")) .setOutputCol("mappings") .setRels(Array("icd10cm_cause", "icd10cm_claim_analysis_code")) val mapper_pipeline = new Pipeline().setStages(Array(document_assembler, chunk_assembler, chunkerMapper)) val data = Seq(Array("D69.51", "G43.83", "A18.03")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +------------+------------------------------------+---------------------------+ |icd10cm_code|cause |icd10cm_claim_analysis_code| +------------+------------------------------------+---------------------------+ |D69.51 |Unintentional injuries |D69.51 | |D69.51 |Adverse effects of medical treatment|D69.51 | |G43.83 |Headache disorders |G43.83 | |G43.83 |Tension-type headache |G43.83 | |G43.83 |Migraine |G43.83 | |A18.03 |Whooping cough |A18.03 | +------------+------------------------------------+---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_cause_claim_mapper| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|600.2 KB| --- layout: model title: Detect bacterial species author: John Snow Labs name: ner_bacterial_species date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect different types of species of bacteria in text using pretrained NER model. ## Predicted Entities `SPECIES` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_BACTERIAL_SPECIES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bacterial_species_en_3.0.0_3.0_1617260641415.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bacterial_species_en_3.0.0_3.0_1617260641415.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_bacterial_species", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_bacterial_species", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.bacterial_species").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bacterial_species| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash +-------+------+-----+-----+------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +-------+------+-----+-----+------+---------+------+------+ |SPECIES|1396.0|265.0|414.0|1810.0| 0.8405|0.7713|0.8044| +-------+------+-----+-----+------+---------+------+------+ +------------------+ | macro| +------------------+ |0.8043791414577931| +------------------+ +------------------+ | micro| +------------------+ |0.8043791414577931| +------------------+ ``` --- layout: model title: English BertForQuestionAnswering model (from piEsposito) author: John Snow Labs name: bert_qa_braquad_bert_qna date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `braquad-bert-qna` is a English model orginally trained by `piEsposito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_braquad_bert_qna_en_4.0.0_3.0_1654537655715.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_braquad_bert_qna_en_4.0.0_3.0_1654537655715.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_braquad_bert_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_braquad_bert_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_piEsposito").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_braquad_bert_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/piEsposito/braquad-bert-qna - https://github.com/piEsposito/br-quad-2.0 --- layout: model title: Pipeline to Detect Genes and Human Phenotypes author: John Snow Labs name: ner_human_phenotype_gene_clinical_pipeline date: 2023-03-15 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_human_phenotype_gene_clinical](https://nlp.johnsnowlabs.com/2021/03/31/ner_human_phenotype_gene_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_clinical_pipeline_en_4.3.0_3.2_1678874928254.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_clinical_pipeline_en_4.3.0_3.2_1678874928254.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_human_phenotype_gene_clinical_pipeline", "en", "clinical/models") text = '''Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_human_phenotype_gene_clinical_pipeline", "en", "clinical/models") val text = "Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3)." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phnotype_gene_clinical.pipeline").predict("""Here we presented a case (BS type) of a 17 years old female presented with polyhydramnios, polyuria, nephrocalcinosis and hypokalemia, which was alleviated after treatment with celecoxib and vitamin D(3).""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_fpdm_roberta_FT_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_fpdm_roberta_FT_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.roberta.qa_fpdm_roberta_ft_newsqa.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_fpdm_roberta_FT_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|458.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/fpdm_roberta_FT_newsqa --- layout: model title: Translate Pohnpeian to English Pipeline author: John Snow Labs name: translate_pon_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, pon, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `pon` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_pon_en_xx_2.7.0_2.4_1609689085209.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_pon_en_xx_2.7.0_2.4_1609689085209.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_pon_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_pon_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.pon.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_pon_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English XLMRobertaForTokenClassification Large Cased model (from asahi417) author: John Snow Labs name: xlmroberta_ner_tner_large_all_english date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tner-xlm-roberta-large-all-english` is a English model originally trained by `asahi417`. ## Predicted Entities `time`, `corporation`, `ordinal number`, `cardinal number`, `rna`, `geopolitical area`, `protein`, `product`, `percent`, `dna`, `disease`, `cell line`, `law`, `other`, `date`, `chemical`, `event`, `work of art`, `cell type`, `location`, `language`, `quantity`, `facility`, `organization`, `group`, `money`, `person` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_all_english_en_4.1.0_3.0_1660424076486.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_all_english_en_4.1.0_3.0_1660424076486.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_all_english","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_all_english","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tner_large_all_english| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.8 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/asahi417/tner-xlm-roberta-large-all-english - https://github.com/asahi417/tner --- layout: model title: Pipeline to Detect Chemicals in Medical Texts author: John Snow Labs name: bert_token_classifier_ner_chemicals_pipeline date: 2022-03-14 tags: [chemicals, ner, bert_token_classifier, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemicals](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemicals_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMICALS/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_2.4_1647264078537.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemicals_pipeline_en_3.4.1_2.4_1647264078537.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python chemicals_pipeline = PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") chemicals_pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` ```scala val chemicals_pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemicals_pipeline", "en", "clinical/models") chemicals_pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemicals_pipeline").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash +---------------------------+---------+ |chunk |ner_label| +---------------------------+---------+ |p - choloroaniline |CHEM | |chlorhexidine - digluconate|CHEM | |kanamycin |CHEM | |colistin |CHEM | |povidone - iodine |CHEM | +---------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.3 MB| ## Included Models - DocumentAssembler - TokenizerModel - MedicalBertForTokenClassifier - NerConverter - Finisher --- layout: model title: Hausa Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_hausa date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, ha, open_source] task: Named Entity Recognition language: ha edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-swahili-finetuned-ner-hausa` is a Hausa model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_hausa_ha_3.4.2_3.0_1652808158883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_hausa_ha_3.4.2_3.0_1652808158883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_swahili_finetuned_ner_hausa| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ha| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-swahili-finetuned-ner-hausa - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman --- layout: model title: English asr_wav2vec2_large_a TFWav2Vec2ForCTC from yongjian author: John Snow Labs name: pipeline_asr_wav2vec2_large_a date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_a` is a English model originally trained by yongjian. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_a_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_a_en_4.2.0_3.0_1664039953191.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_a_en_4.2.0_3.0_1664039953191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_a', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_a", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_a| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Smaller BERT Embeddings (L-12_H-256_A-4) author: John Snow Labs name: small_bert_L12_256 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L12_256_en_2.6.0_2.4_1598344517363.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L12_256_en_2.6.0_2.4_1598344517363.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L12_256", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L12_256", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L12_256').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L12_256_embeddings I [2.3376450538635254, -1.8108669519424438, 0.00... love [0.369129478931427, -0.1494956910610199, 0.391... NLP [0.31006014347076416, 0.20426464080810547, -0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L12_256| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1 --- layout: model title: Translate Cushitic languages to English Pipeline author: John Snow Labs name: translate_cus_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, cus, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `cus` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_cus_en_xx_2.7.0_2.4_1609690727352.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_cus_en_xx_2.7.0_2.4_1609690727352.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_cus_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_cus_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.cus.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_cus_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes (sbiobertresolve_icd10cm_slim_billable_hcc) author: John Snow Labs name: sbiobertresolve_icd10cm_slim_billable_hcc date: 2022-05-11 tags: [licensed, en, clinical, icd10, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted clinical entities to ICD-10-CM codes using `sbiobert_base_cased_mli` sentence bert embeddings. In this model, synonyms having low cosine similarity to unnormalized terms are dropped. It returns the official resolution text within the brackets and also provides billable and HCC information of the codes in `all_k_aux_labels` parameter in the metadata. This column can be divided to get further details: `billable status || hcc status || hcc score`. For example, if `all_k_aux_labels` is like `1||1||19` which means the `billable status` is 1, `hcc status` is 1, and `hcc score` is 19. ## Predicted Entities `ICD10 Codes`, `billable status`, `hcc status`, `hcc score` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.5.1_3.0_1652294908790.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.5.1_3.0_1652294908790.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+--------+-------------------------------------------------------------------+--------------------------------------------------+-------------------------------------------------------+ | chunk| entity|icd_code| all_k_resolutions| all_k_codes| all_k_aux_labels | +-------------------------------------+-------+--------+-------------------------------------------------------------------+--------------------------------------------------+-------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.41|gestational diabetes mellitus [Gestational diabetes mellitus in ...|O24.41:::E11.9:::O24.919:::O24.419:::O24.439:::...| 0||0||0:::1||1||19:::1||0||0:::1||0||0:::1||0||0:::...| |subsequent type two diabetes mellitus|PROBLEM| O24.11|pre-existing type 2 diabetes mellitus [Pre-existing type 2 diabe...|O24.11:::E11.8:::E11.9:::E11:::E13.9:::E11.3:::...| 0||0||0:::1||1||18:::1||1||19:::0||0||0:::1||1||19:...| | T2DM|PROBLEM| E11.9|t2dm [Type 2 diabetes mellitus without complications]:::gm>2 [GM...|E11.9:::E75.00:::H35.89:::F80.0:::R44.8:::M79.8...| 1||1||19:::1||1||52:::1||0||0:::1||0||0:::1||0||0::...| | HTG-induced pancreatitis|PROBLEM| K85.9|alcohol-induced pancreatitis [Acute pancreatitis, unspecified]:::..|K85.9:::F10.988:::K85.3:::K85:::K85.2:::K85.8::...| 0||0||0:::1||1||55:::0||0||0:::0||0||0:::0||0||0:::...| | acute hepatitis|PROBLEM| K72.0|acute hepatitis [Acute and subacute hepatic failure]:::acute hep...|K72.0:::B17.9:::B15.9:::B15:::B17.2:::Z03.89:::...| 0||0||0:::1||0||0:::1||0||0:::0||0||0:::1||0||0:::1...| | obesity|PROBLEM| E66.8|abdominal obesity [Other obesity]:::overweight and obesity [Over...|E66.8:::E66:::E66.01:::E66.9:::Z91.89:::E66.3::...| 1||0||0:::0||0||0:::1||1||22:::1||0||0:::1||0||0:::...| | polyuria|PROBLEM| R35|polyuria [Polyuria]:::nocturnal polyuria [Nocturnal polyuria]:::...|R35:::R35.81:::R35.89:::R31:::R30.0:::E72.01:::...| 0||0||0:::1||0||0:::1||0||0:::0||0||0:::1||0||0:::1...| | polydipsia|PROBLEM| R63.1|polydipsia [Polydipsia]:::psychogenic polydipsia [Other impulse ...|R63.1:::F63.89:::O40:::O40.9XX0:::G47.50:::G47....| 1||0||0:::1||0||0:::0||0||0:::1||0||0:::1||0||0:::0...| | poor appetite|PROBLEM| R63.0|poor appetite [Anorexia]:::patient dissatisfied with nutrition r...|R63.0:::Z76.89:::R53.1:::R10.9:::R45.81:::R44.8...| 1||0||0:::1||0||0:::1||0||0:::1||0||0:::1||0||0:::1...| | vomiting|PROBLEM| R11.1|vomiting [Vomiting]:::vomiting [Vomiting, unspecified]:::intermi...|R11.1:::R11.10:::R11:::G43.A0:::G43.A:::R11.0::...| 0||0||0:::1||0||0:::0||0||0:::1||0||0:::0||0||0:::1...| | a respiratory tract infection|PROBLEM| J06.9|upper respiratory tract infection [Acute upper respiratory infec...|J06.9:::T17.9:::T17:::J04.10:::J22:::J98.8:::J9...| 1||0||0:::0||0||0:::0||0||0:::1||0||0:::1||0||0:::1...| +-------------------------------------+-------+--------+----------------------------------------------------------------------------------------------------+--------------------------------------------------+-------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|846.6 MB| |Case sensitive:|false| --- layout: model title: Public Health Surveillance (PHS) BERT Embeddings author: John Snow Labs name: bert_embeddings_phs_bert date: 2022-07-02 tags: [bert, en, embeddings, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BERT Embeddings model, adapted from [Hugging Face](https://huggingface.co/publichealthsurveillance/PHS-BERT) and curated to provide scalability and production-readiness using Spark NLP. `PHS-BERT` is an English model and trained to identify the tasks related to public health surveillance (PHS) on social media. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_phs_bert_en_4.0.0_3.0_1656759538082.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_phs_bert_en_4.0.0_3.0_1656759538082.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_phs_bert| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| ## References https://arxiv.org/abs/2204.04521 --- layout: model title: Detect actions in general commands related to music, restaurant, movies. author: John Snow Labs name: nerdl_snips_100d date: 2021-02-15 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.7.3 spark_version: 2.4 tags: [open_source, ner, en] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Understand user commands and find relevant entities and actions and tag them to get a structured representation for automation. ## Predicted Entities `playlist_owner`, `served_dish`, `track`, `poi`, `cuisine`, `spatial_relation`, `object_type`, `facility`, `album`, `country`, `geographic_poi`, `location_name`, `object_part_of_series_type`, `object_select`, `artist`, `rating_value`, `best_rating`, `sort`, `party_size_description`, `party_size_number`, `restaurant_name`, `object_location_type`, `playlist`, `service`, `city`, `O`, `genre`, `movie_name`, `current_location`, `rating_unit`, `restaurant_type`, `condition_temperature`, `condition_description`, `entity_name`, `movie_type`, `object_name`, `state`, `year`, `music_item`, `timeRange` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_CLS_SNIPS){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_snips_100d_en_2.7.3_2.4_1613403676821.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_snips_100d_en_2.7.3_2.4_1613403676821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+------------------------+ | ner_chunk | label | +------------------------------+------------------------+ | nona gray myrtle and alison | PARTY_SIZE_DESCRIPTION | | top-rated | SORT | | brasserie | RESTAURANT_TYPE | | distant | SPATIAL_RELATION | | wilson Macro-average | POI | | nov the 4th 2030 | TIMERANGE | | ouzeri | CUISINE | +------------------------------+------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_snips_100d| |Type:|ner| |Compatibility:|Spark NLP 2.7.3+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source This model is trained on the NLU Benchmark SNIPS dataset https://github.com/MiuLab/SlotGated-SLU ## Benchmarking ```bash B-facility 3 0 0 1.0 1.0 1.0 B-poi 7 0 1 1.0 0.875 0.93333334 B-object_location_type 22 1 0 0.95652175 1.0 0.9777778 B-service 24 2 0 0.9230769 1.0 0.96000004 I-entity_name 53 2 1 0.96363634 0.9814815 0.9724771 B-genre 5 0 0 1.0 1.0 1.0 I-service 5 0 0 1.0 1.0 1.0 I-object_type 66 0 0 1.0 1.0 1.0 I-sort 9 0 0 1.0 1.0 1.0 I-city 19 1 0 0.95 1.0 0.9743589 B-music_item 102 2 2 0.9807692 0.9807692 0.9807692 I-movie_name 100 5 21 0.95238096 0.8264463 0.8849558 B-party_size_description 10 0 0 1.0 1.0 1.0 B-served_dish 10 3 2 0.7692308 0.8333333 0.8 B-object_type 161 8 1 0.9526627 0.99382716 0.9728097 B-playlist 123 6 6 0.95348835 0.95348835 0.95348835 B-restaurant_name 14 1 1 0.93333334 0.93333334 0.93333334 B-geographic_poi 11 0 0 1.0 1.0 1.0 B-condition_description 28 0 0 1.0 1.0 1.0 I-object_location_type 16 0 0 1.0 1.0 1.0 B-spatial_relation 70 3 1 0.9589041 0.9859155 0.9722222 I-party_size_description 35 0 0 1.0 1.0 1.0 I-poi 10 0 1 1.0 0.90909094 0.95238096 I-artist 111 4 1 0.9652174 0.9910714 0.9779735 B-condition_temperature 23 0 0 1.0 1.0 1.0 I-movie_type 16 0 0 1.0 1.0 1.0 I-object_part_of_series_type 0 0 1 0.0 0.0 0.0 B-city 60 1 0 0.9836066 1.0 0.9917355 I-location_name 29 0 1 1.0 0.96666664 0.9830508 B-album 0 2 10 0.0 0.0 0.0 I-genre 2 0 0 1.0 1.0 1.0 B-state 55 0 4 1.0 0.9322034 0.9649123 I-object_name 383 29 16 0.9296116 0.9598997 0.9445129 B-current_location 13 0 1 1.0 0.9285714 0.9629629 B-timeRange 102 8 5 0.92727274 0.95327103 0.9400922 B-sort 29 1 3 0.96666664 0.90625 0.9354838 I-timeRange 144 7 0 0.95364237 1.0 0.97627115 B-rating_unit 40 0 0 1.0 1.0 1.0 I-current_location 7 0 0 1.0 1.0 1.0 I-state 6 0 0 1.0 1.0 1.0 I-album 4 1 17 0.8 0.1904762 0.30769232 B-entity_name 31 4 2 0.8857143 0.93939394 0.9117647 B-object_name 134 22 13 0.85897434 0.91156465 0.88448846 B-playlist_owner 70 1 0 0.9859155 1.0 0.9929078 I-music_item 5 0 0 1.0 1.0 1.0 I-spatial_relation 41 2 1 0.95348835 0.97619045 0.9647058 I-country 25 1 0 0.96153843 1.0 0.98039216 B-rating_value 80 0 0 1.0 1.0 1.0 B-restaurant_type 64 0 1 1.0 0.9846154 0.9922481 I-playlist_owner 7 0 0 1.0 1.0 1.0 I-cuisine 1 0 0 1.0 1.0 1.0 B-track 7 10 2 0.4117647 0.7777778 0.5384615 B-movie_name 37 2 10 0.94871795 0.78723407 0.8604651 B-party_size_number 50 0 0 1.0 1.0 1.0 I-restaurant_type 7 0 0 1.0 1.0 1.0 B-year 24 1 0 0.96 1.0 0.9795918 B-location_name 23 0 1 1.0 0.9583333 0.9787234 B-object_part_of_series_type 11 1 0 0.9166667 1.0 0.95652175 B-country 43 4 1 0.9148936 0.97727275 0.94505495 I-playlist 218 4 13 0.981982 0.94372296 0.96247244 I-served_dish 2 1 2 0.6666667 0.5 0.57142854 I-track 19 29 2 0.39583334 0.9047619 0.5507246 B-artist 99 4 8 0.9611651 0.92523366 0.9428571 B-best_rating 43 0 0 1.0 1.0 1.0 I-restaurant_name 35 2 1 0.9459459 0.9722222 0.9589041 B-object_select 40 1 0 0.9756098 1.0 0.9876543 B-cuisine 12 1 2 0.9230769 0.85714287 0.8888889 B-movie_type 33 0 0 1.0 1.0 1.0 I-geographic_poi 33 0 0 1.0 1.0 1.0 tp: 3121 fp: 177 fn: 155 labels: 69 Macro-average prec: 0.91982585, rec: 0.9205297, f1: 0.9201776 Micro-average prec: 0.9463311, rec: 0.9526862, f1: 0.949498 ``` --- layout: model title: Multilingual T5ForConditionalGeneration Small Cased model (from north) author: John Snow Labs name: t5_small_ncc_lm date: 2023-01-31 tags: [is, nn, en, "no", sv, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5_small_NCC_lm` is a Multilingual model originally trained by `north`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_ncc_lm_xx_4.3.0_3.0_1675157076420.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_ncc_lm_xx_4.3.0_3.0_1675157076420.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_ncc_lm","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_ncc_lm","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_ncc_lm| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|1.6 GB| ## References - https://huggingface.co/north/t5_small_NCC_lm - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/google-research/t5x - https://arxiv.org/abs/2104.09617 - https://arxiv.org/abs/2104.09617 - https://arxiv.org/pdf/1910.10683.pdf - https://sites.research.google/trc/about/ --- layout: model title: Pipeline to Extract entities in clinical trial abstracts (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_clinical_trials_abstracts_pipeline date: 2023-03-20 tags: [berttokenclassifier, bert, biobert, en, ner, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_clinical_trials_abstracts](https://nlp.johnsnowlabs.com/2022/06/29/bert_token_classifier_ner_clinical_trials_abstracts_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_pipeline_en_4.3.0_3.2_1679304059319.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_clinical_trials_abstracts_pipeline_en_4.3.0_3.2_1679304059319.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_clinical_trials_abstracts_pipeline", "en", "clinical/models") text = '''This open-label, parallel-group, two-arm, pilot study compared the beta-cell protective effect of adding insulin glargine (GLA) vs. NPH insulin to ongoing metformin. Overall, 28 insulin-naive type 2 diabetes subjects (mean +/- SD age, 61.5 +/- 6.7 years; BMI, 30.7 +/- 4.3 kg/m(2)) treated with metformin and sulfonylurea were randomized to add once-daily GLA or NPH at bedtime.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_clinical_trials_abstracts_pipeline", "en", "clinical/models") val text = "This open-label, parallel-group, two-arm, pilot study compared the beta-cell protective effect of adding insulin glargine (GLA) vs. NPH insulin to ongoing metformin. Overall, 28 insulin-naive type 2 diabetes subjects (mean +/- SD age, 61.5 +/- 6.7 years; BMI, 30.7 +/- 4.3 kg/m(2)) treated with metformin and sulfonylurea were randomized to add once-daily GLA or NPH at bedtime." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:-------------------|-------------:| | 0 | open-label | 5 | 14 | CTDesign | 0.742075 | | 1 | parallel-group | 17 | 30 | CTDesign | 0.725741 | | 2 | two-arm | 33 | 39 | CTDesign | 0.427547 | | 3 | insulin glargine | 105 | 120 | Drug | 0.985063 | | 4 | GLA | 123 | 125 | Drug | 0.96917 | | 5 | NPH insulin | 132 | 142 | Drug | 0.762519 | | 6 | metformin | 155 | 163 | Drug | 0.996344 | | 7 | 28 | 175 | 176 | NumberPatients | 0.968501 | | 8 | type 2 diabetes | 192 | 206 | DisorderOrSyndrome | 0.979685 | | 9 | 61.5 | 235 | 238 | Age | 0.610416 | | 10 | kg/m(2 | 273 | 278 | BioAndMedicalUnit | 0.974807 | | 11 | metformin | 295 | 303 | Drug | 0.99696 | | 12 | sulfonylurea | 309 | 320 | Drug | 0.996722 | | 13 | randomized | 327 | 336 | CTDesign | 0.990632 | | 14 | once-daily | 345 | 354 | DrugTime | 0.472084 | | 15 | GLA | 356 | 358 | Drug | 0.972978 | | 16 | NPH | 363 | 365 | Drug | 0.989424 | | 17 | bedtime | 370 | 376 | DrugTime | 0.936016 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_clinical_trials_abstracts_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.9 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from huxxx657) author: John Snow Labs name: roberta_qa_huxxx657_base_finetuned_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_huxxx657_base_finetuned_squad_en_4.3.0_3.0_1674217238547.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_huxxx657_base_finetuned_squad_en_4.3.0_3.0_1674217238547.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_huxxx657_base_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_huxxx657_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_huxxx657_base_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/huxxx657/roberta-base-finetuned-squad --- layout: model title: English RobertaForQuestionAnswering (from AlirezaBaneshi) author: John Snow Labs name: roberta_qa_autotrain_test2_756523214 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-test2-756523214` is a English model originally trained by `AlirezaBaneshi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523214_en_4.0.0_3.0_1655727699500.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_autotrain_test2_756523214_en_4.0.0_3.0_1655727699500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_autotrain_test2_756523214","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_autotrain_test2_756523214","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.756523214.by_AlirezaBaneshi").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_autotrain_test2_756523214| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|415.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AlirezaBaneshi/autotrain-test2-756523214 --- layout: model title: Pipeline to Detect Radiology Concepts (WIP Greedy) author: John Snow Labs name: jsl_rd_ner_wip_greedy_clinical_pipeline date: 2022-03-21 tags: [licensed, ner, wip, clinical, radiology, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [jsl_rd_ner_wip_greedy_clinical](https://nlp.johnsnowlabs.com/2021/04/01/jsl_rd_ner_wip_greedy_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_clinical_pipeline_en_3.4.1_3.0_1647867631313.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_clinical_pipeline_en_3.4.1_3.0_1647867631313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("jsl_rd_ner_wip_greedy_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.") ``` ```scala val pipeline = new PretrainedPipeline("jsl_rd_ner_wip_greedy_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_rd_wip_greedy.pipeline").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash +---------------------+-------------------------+ |chunks |entities | +---------------------+-------------------------+ |Bilateral |Direction | |breast |BodyPart | |ultrasound |ImagingTest | |ovoid mass |Symptom | |anteromedial aspect |Direction | |left |Direction | |shoulder |BodyPart | |mass |Symptom | |isoechoic echotexture|ImagingFindings | |adjacent muscle |BodyPart | |internal color flow |Symptom | |benign fibrous tissue|Symptom | |lipoma |Disease_Syndrome_Disorder| +---------------------+-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_rd_ner_wip_greedy_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Arabic Bert Embeddings (Base, Trained on a half of the full MSA dataset) author: John Snow Labs name: bert_embeddings_bert_base_arabic_camelbert_msa_half date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-msa-half` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_half_ar_3.4.2_3.0_1649678623023.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_half_ar_3.4.2_3.0_1649678623023.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabic_camelbert_msa_half| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-half - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: Legal Administration Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_administration_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, administration, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_administration_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `administration-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `administration-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_administration_agreement_bert_en_1.0.0_3.0_1669300445143.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_administration_agreement_bert_en_1.0.0_3.0_1669300445143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_administration_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_elasticdotventures_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_elasticdotventures_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_elasticdotventures_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/elasticdotventures/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Term of agreement Clause Binary Classifier (md) author: John Snow Labs name: legclf_term_of_agreement_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `term-of-agreement` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `term-of-agreement` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_term_of_agreement_md_en_1.0.0_3.0_1673460298428.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_term_of_agreement_md_en_1.0.0_3.0_1673460298428.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_term_of_agreement_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[term-of-agreement]| |[other]| |[other]| |[term-of-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_term_of_agreement_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support other 0.90 0.97 0.94 39 tax-matters 0.97 0.89 0.93 35 accuracy 0.93 74 macro avg 0.94 0.93 0.93 74 weighted avg 0.94 0.93 0.93 74 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from lewtun) author: John Snow Labs name: bert_qa_lewtun_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `lewtun`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_lewtun_finetuned_squad_en_4.0.0_3.0_1657186592485.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_lewtun_finetuned_squad_en_4.0.0_3.0_1657186592485.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_lewtun_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_lewtun_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_lewtun_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/lewtun/bert-finetuned-squad --- layout: model title: Legal Criticality Prediction Classifier (Italian) author: John Snow Labs name: legclf_critical_prediction_italian date: 2023-03-27 tags: [it, licensed, legal, classification, tensorflow] task: Text Classification language: it edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Description: This is a Binary classification model which identifies two criticality labels(critical, non-critical) in Italian-based Court Cases. ## Predicted Entities `critical`, `non-critical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_critical_prediction_italian_it_1.0.0_3.0_1679944691458.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_critical_prediction_italian_it_1.0.0_3.0_1679944691458.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = nlp.RoBertaForSequenceClassification.pretrained("legclf_critical_prediction_italian", "it", "legal/models") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") nlpPipeline = nlp.Pipeline( stages = [documentAssembler, tokenizer, classifier]) # Example text example = spark.createDataFrame([["Per questi motivi, il Tribunale federale pronuncia: 1. Nella misura in cui è ammissibile, il ricorso è respinto. 2. Le spese giudiziarie di fr. 2'000.-- sono poste a carico del ricorrente. 3. Comunicazione ai patrocinatori delle parti, al patrocinatore di C._ e al Presidente della Camera di protezione del Tribunale d'appello del Cantone Ticino."]]).toDF("text") empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) result = model.transform(example) # result is a DataFrame result.select("text", "class.result").show(truncate=100) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+----------+ | text| result| +----------------------------------------------------------------------------------------------------+----------+ |Per questi motivi, il Tribunale federale pronuncia: 1. Nella misura in cui è ammissibile, il rico...|[critical]| +----------------------------------------------------------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_critical_prediction_italian| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|it| |Size:|415.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References Train dataset available [here](https://huggingface.co/datasets/rcds/legal_criticality_prediction) ## Benchmarking ```bash label precision recall f1-score support critical 0.82 0.90 0.86 10 non_critical 0.95 0.91 0.93 23 accuracy - - 0.91 33 macro-avg 0.89 0.91 0.90 33 weighted-avg 0.91 0.91 0.91 33 ``` --- layout: model title: Arabic Bert Embeddings (Base, Trained on a quarter of the full MSA dataset) author: John Snow Labs name: bert_embeddings_bert_base_arabic_camelbert_msa_quarter date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-msa-quarter` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_quarter_ar_3.4.2_3.0_1649678578942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_arabic_camelbert_msa_quarter_ar_3.4.2_3.0_1649678578942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_arabic_camelbert_msa_quarter| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|409.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-quarter - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://catalog.ldc.upenn.edu/LDC2011T11 - http://www.abuelkhair.net/index.php/en/arabic/abu-el-khair-corpus - https://vlo.clarin.eu/search;jsessionid=31066390B2C9E8C6304845BA79869AC1?1&q=osian - https://archive.org/details/arwiki-20190201 - https://oscar-corpus.com/ - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/eedf5716ce1268e56f0a50264a88cafad334ac61/tokenization.py#L286-L297 - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/CAMeLBERT --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_900000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-900000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_900000_cased_generator_de_3.4.4_3.0_1652786530316.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_900000_cased_generator_de_3.4.4_3.0_1652786530316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_900000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-900000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: Tagalog Electra Embeddings (from jcblaise) author: John Snow Labs name: electra_embeddings_electra_tagalog_base_uncased_generator date: 2022-05-17 tags: [tl, open_source, electra, embeddings] task: Embeddings language: tl edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-tagalog-base-uncased-generator` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_base_uncased_generator_tl_3.4.4_3.0_1652786747701.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_base_uncased_generator_tl_3.4.4_3.0_1652786747701.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_tagalog_base_uncased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tl| |Size:|130.6 MB| |Case sensitive:|false| ## References - https://huggingface.co/jcblaise/electra-tagalog-base-uncased-generator - https://blaisecruz.com --- layout: model title: German RoBERTa Embeddings (Small, Hotel) author: John Snow Labs name: roberta_embeddings_HotelBERT_small date: 2022-04-14 tags: [roberta, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `HotelBERT-small` is a German model orginally trained by `FabianGroeger`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_HotelBERT_small_de_3.4.2_3.0_1649948046264.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_HotelBERT_small_de_3.4.2_3.0_1649948046264.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_HotelBERT_small| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|313.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/FabianGroeger/HotelBERT-small --- layout: model title: Legal Successors Clause Binary Classifier author: John Snow Labs name: legclf_successors_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `successors` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `successors` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_successors_clause_en_1.0.0_3.2_1660124037330.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_successors_clause_en_1.0.0_3.2_1660124037330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_successors_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[successors]| |[other]| |[other]| |[successors]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_successors_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.98 0.99 159 successors 0.96 1.00 0.98 72 accuracy - - 0.99 231 macro-avg 0.98 0.99 0.99 231 weighted-avg 0.99 0.99 0.99 231 ``` --- layout: model title: Sentence Entity Resolver for UMLS CUI Codes author: John Snow Labs name: sbiobertresolve_umls_findings date: 2022-07-05 tags: [entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to 4 major categories of UMLS CUI codes using sbiobert_base_cased_mli Sentence Bert Embeddings. It has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. This model returns CUI (concept unique identifier) codes for 200K concepts from clinical findings.https://www.nlm.nih.gov/research/umls/index.html ## Predicted Entities `umls cuı codes` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_UMLS_CUI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_4.0.0_3.0_1657041603500.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_umls_findings_en_4.0.0_3.0_1657041603500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("clinical_ner") ner_model_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "clinical_ner"])\ .setOutputCol("ner_chunk") chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli",'en','clinical/models')\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_umls_findings","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline = Pipeline(stages = [document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunk2doc, sbert_embedder, resolver]) data = spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting."""]]).toDF("text") results = pipeline.fit(data).transform(data) ``` ```scala ... val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel .pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("clinical_ner") val ner_model_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "clinical_ner")) .setOutputCol("ner_chunk") val chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_umls_findings", "en", "clinical/models") .setInputCols(Array("ner_chunk_doc", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val p_model = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner_model, ner_model_converter, chunk2doc, sbert_embedder, resolver)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting.").toDF("text") val res = p_model.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.umls.findings").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_4", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_4", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_4| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Lemmatizer (English, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-08 tags: [open_source, lemmatizer, en] task: Lemmatization language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This English Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_en_3.4.1_3.0_1646753615538.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_en_3.4.1_3.0_1646753615538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+ |result | +------------------------------+ |[You, be, not, well, than, me]| +------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|en| |Size:|427.0 KB| --- layout: model title: Legal Fiscal year Clause Binary Classifier author: John Snow Labs name: legclf_fiscal_year_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `fiscal-year` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `fiscal-year` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_fiscal_year_clause_en_1.0.0_3.2_1660123548661.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_fiscal_year_clause_en_1.0.0_3.2_1660123548661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_fiscal_year_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[fiscal-year]| |[other]| |[other]| |[fiscal-year]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_fiscal_year_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support fiscal-year 1.00 0.96 0.98 45 other 0.98 1.00 0.99 91 accuracy - - 0.99 136 macro-avg 0.99 0.98 0.98 136 weighted-avg 0.99 0.99 0.99 136 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Bislama author: John Snow Labs name: opus_mt_en_bi date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bi, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bi_xx_2.7.0_2.4_1609170461176.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bi_xx_2.7.0_2.4_1609170461176.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bi| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739695554.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739695554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_twostage_quadruplet_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|306.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_twostage_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-6` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6_en_4.3.0_3.0_1674215648700.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6_en_4.3.0_3.0_1674215648700.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_512_finetuned_squad_seed_6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|432.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-6 --- layout: model title: English DistilBertForMaskedLM Base Uncased model (from Intel) author: John Snow Labs name: distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa date: 2022-12-12 tags: [en, open_source, distilbert_embeddings, distilbertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-sparse-90-unstructured-pruneofa` is a English model originally trained by `Intel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa_en_4.2.4_3.0_1670864808665.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa_en_4.2.4_3.0_1670864808665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(False) pipeline = Pipeline(stages=[documentAssembler, tokenizer, distilbert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(false) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, distilbert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_base_uncased_sparse_90_unstructured_pruneofa| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|123.6 MB| |Case sensitive:|false| ## References - https://huggingface.co/Intel/distilbert-base-uncased-sparse-90-unstructured-pruneofa - https://arxiv.org/abs/2111.05754 - https://github.com/IntelLabs/Model-Compression-Research-Package/tree/main/research/prune-once-for-all --- layout: model title: English RobertaForQuestionAnswering (from prk) author: John Snow Labs name: roberta_qa_prk_roberta_base_squad2_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `prk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_prk_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735480993.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_prk_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735480993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_prk_roberta_base_squad2_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_prk_roberta_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_prk").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_prk_roberta_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/prk/roberta-base-squad2-finetuned-squad --- layout: model title: Lemmatizer (Hungarian, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, hu] task: Lemmatization language: hu edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Hungarian Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_hu_3.4.1_3.0_1646316501230.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_hu_3.4.1_3.0_1646316501230.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+ |result | +---------------------------+ |[Nem, van, jó, ,, mint, én]| +---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|hu| |Size:|440.7 KB| --- layout: model title: English T5ForConditionalGeneration Small Cased model (from ml6team) author: John Snow Labs name: t5_keyphrase_generation_small_inspec date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `keyphrase-generation-t5-small-inspec` is a English model originally trained by `ml6team`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_keyphrase_generation_small_inspec_en_4.3.0_3.0_1675104684365.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_keyphrase_generation_small_inspec_en_4.3.0_3.0_1675104684365.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_keyphrase_generation_small_inspec","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_keyphrase_generation_small_inspec","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_keyphrase_generation_small_inspec| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|280.5 MB| ## References - https://huggingface.co/ml6team/keyphrase-generation-t5-small-inspec - https://dl.acm.org/doi/10.3115/1119355.1119383 - https://paperswithcode.com/sota?task=Keyphrase+Generation&dataset=inspec --- layout: model title: Danish asr_wav2vec2_base_nst TFWav2Vec2ForCTC from Alvenir author: John Snow Labs name: asr_wav2vec2_base_nst date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_nst` is a Danish model originally trained by Alvenir. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_nst_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_nst_da_4.2.0_3.0_1664103793330.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_nst_da_4.2.0_3.0_1664103793330.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_nst", "da")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_nst", "da") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_nst| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|da| |Size:|354.3 MB| --- layout: model title: Spanish RoBERTa Embeddings (Base) author: John Snow Labs name: roberta_embeddings_roberta_base_bne date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-base-bne` is a Spanish model orginally trained by `PlanTL-GOB-ES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_base_bne_es_3.4.2_3.0_1649944766661.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_base_bne_es_3.4.2_3.0_1649944766661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_base_bne| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|297.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - http://www.bne.es/en/Inicio/index.html - https://arxiv.org/abs/1907.11692 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Stop Words Cleaner for Bulgarian author: John Snow Labs name: stopwords_bg date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: bg edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, bg] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_bg_bg_2.5.4_2.4_1594742440962.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_bg_bg_2.5.4_2.4_1594742440962.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_bg", "bg") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_bg", "bg") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Освен че е крал на север, Джон Сноу е английски лекар и лидер в развитието на анестезия и медицинска хигиена."""] stopword_df = nlu.load('bg.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=11, end=14, result='крал', metadata={'sentence': '0'}), Row(annotatorType='token', begin=19, end=23, result='север', metadata={'sentence': '0'}), Row(annotatorType='token', begin=24, end=24, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=26, end=29, result='Джон', metadata={'sentence': '0'}), Row(annotatorType='token', begin=31, end=34, result='Сноу', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_bg| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|bg| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English XlmRoBertaForQuestionAnswering (from alon-albalak) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_base_xquad date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-xquad` is a English model originally trained by `alon-albalak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_xquad_en_4.0.0_3.0_1655991956893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_base_xquad_en_4.0.0_3.0_1655991956893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_base_xquad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_base_xquad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xquad.xlm_roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_base_xquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|847.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/alon-albalak/xlm-roberta-base-xquad - https://github.com/deepmind/xquad --- layout: model title: Arabic BertForTokenClassification Cased model (from abdusah) author: John Snow Labs name: bert_token_classifier_ara_ner date: 2022-11-30 tags: [ar, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `arabert-ner` is a Arabic model originally trained by `abdusah`. ## Predicted Entities `PER`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ara_ner_ar_4.2.4_3.0_1669814157826.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ara_ner_ar_4.2.4_3.0_1669814157826.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ara_ner","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_ara_ner","ar") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ara_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|ar| |Size:|505.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/abdusah/arabert-ner --- layout: model title: Pipeline to Summarize Clinical Notes (PubMed) author: John Snow Labs name: summarizer_biomedical_pubmed_pipeline date: 2023-05-29 tags: [licensed, en, clinical, text_summarization, biomedical] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_biomedical_pubmed](https://nlp.johnsnowlabs.com/2023/04/03/summarizer_biomedical_pubmed_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_biomedical_pubmed_pipeline_en_4.4.2_3.0_1685399522856.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_biomedical_pubmed_pipeline_en_4.4.2_3.0_1685399522856.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_biomedical_pubmed_pipeline", "en", "clinical/models") text = """Residual disease after initial surgery for ovarian cancer is the strongest prognostic factor for survival. However, the extent of surgical resection required to achieve optimal cytoreduction is controversial. Our goal was to estimate the effect of aggressive surgical resection on ovarian cancer patient survival.\n A retrospective cohort study of consecutive patients with International Federation of Gynecology and Obstetrics stage IIIC ovarian cancer undergoing primary surgery was conducted between January 1, 1994, and December 31, 1998. The main outcome measures were residual disease after cytoreduction, frequency of radical surgical resection, and 5-year disease-specific survival.\n The study comprised 194 patients, including 144 with carcinomatosis. The mean patient age and follow-up time were 64.4 and 3.5 years, respectively. After surgery, 131 (67.5%) of the 194 patients had less than 1 cm of residual disease (definition of optimal cytoreduction). Considering all patients, residual disease was the only independent predictor of survival; the need to perform radical procedures to achieve optimal cytoreduction was not associated with a decrease in survival. For the subgroup of patients with carcinomatosis, residual disease and the performance of radical surgical procedures were the only independent predictors. Disease-specific survival was markedly improved for patients with carcinomatosis operated on by surgeons who most frequently used radical procedures compared with those least likely to use radical procedures (44% versus 17%, P < .001).\n Overall, residual disease was the only independent predictor of survival. Minimizing residual disease through aggressive surgical resection was beneficial, especially in patients with carcinomatosis. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_biomedical_pubmed_pipeline", "en", "clinical/models") val text = """Residual disease after initial surgery for ovarian cancer is the strongest prognostic factor for survival. However, the extent of surgical resection required to achieve optimal cytoreduction is controversial. Our goal was to estimate the effect of aggressive surgical resection on ovarian cancer patient survival.\n A retrospective cohort study of consecutive patients with International Federation of Gynecology and Obstetrics stage IIIC ovarian cancer undergoing primary surgery was conducted between January 1, 1994, and December 31, 1998. The main outcome measures were residual disease after cytoreduction, frequency of radical surgical resection, and 5-year disease-specific survival.\n The study comprised 194 patients, including 144 with carcinomatosis. The mean patient age and follow-up time were 64.4 and 3.5 years, respectively. After surgery, 131 (67.5%) of the 194 patients had less than 1 cm of residual disease (definition of optimal cytoreduction). Considering all patients, residual disease was the only independent predictor of survival; the need to perform radical procedures to achieve optimal cytoreduction was not associated with a decrease in survival. For the subgroup of patients with carcinomatosis, residual disease and the performance of radical surgical procedures were the only independent predictors. Disease-specific survival was markedly improved for patients with carcinomatosis operated on by surgeons who most frequently used radical procedures compared with those least likely to use radical procedures (44% versus 17%, P < .001).\n Overall, residual disease was the only independent predictor of survival. Minimizing residual disease through aggressive surgical resection was beneficial, especially in patients with carcinomatosis. """ val result = pipeline.fullAnnotate(text) ```

## Results ```bash The results of this review suggest that aggressive ovarian cancer surgery is associated with a significant reduction in the risk of recurrence and a reduction in the number of radical versus conservative surgical resections. However, the results of this review are based on only one small trial. Further research is needed to determine the role of aggressive ovarian cancer surgery in women with stage IIIC ovarian cancer. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_biomedical_pubmed_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|936.7 MB| ## Included Models - DocumentAssembler - MedicalSummarizer --- layout: model title: Legal Anti Corruption Laws Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_anti_corruption_laws_bert date: 2023-03-05 tags: [en, legal, classification, clauses, anti_corruption_laws, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Anti_Corruption_Laws` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Anti_Corruption_Laws`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_anti_corruption_laws_bert_en_1.0.0_3.0_1678049923672.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_anti_corruption_laws_bert_en_1.0.0_3.0_1678049923672.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_anti_corruption_laws_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Anti_Corruption_Laws]| |[Other]| |[Other]| |[Anti_Corruption_Laws]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_anti_corruption_laws_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Anti_Corruption_Laws 1.00 1.00 1.0 11 Other 1.00 1.00 1.0 19 accuracy - - 1.0 30 macro-avg 1.00 1.00 1.0 30 weighted-avg 1.00 1.00 1.0 30 ``` --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_12_h_128 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-12_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_128_zh_4.2.4_3.0_1670325790515.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_128_zh_4.2.4_3.0_1670325790515.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_2_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_2_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_2_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|117.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-2_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Detect Clinical Entities, Assign Assertion and Find Relations author: John Snow Labs name: explain_clinical_doc_era date: 2020-09-30 task: [Named Entity Recognition, Assertion Status, Relation Extraction, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 2.6.0 spark_version: 2.4 tags: [pipeline, en, licensed, clinical] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained pipeline with ``ner_clinical_events``, ``assertion_dl`` and ``re_temporal_events_clinical`` trained with ``embeddings_healthcare_100d``. It will extract clinical entities, assign assertion status and find temporal relationships between clinical entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_2.5.5_2.4_1597845753750.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_era_en_2.5.5_2.4_1597845753750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python era_pipeline = PretrainedPipeline('explain_clinical_doc_era', 'en', 'clinical/models') annotations = era_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)[0] annotations.keys() ``` ```scala val era_pipeline = new PretrainedPipeline("explain_clinical_doc_era", "en", "clinical/models") val result = era_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)(0) ``` {:.nlu-block} ```python import nlu nlu.load("en.explain_doc.era").predict("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache. She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """) ```

{:.h2_title} ## Results The output is a dictionary with the following keys: ``'sentences'``, ``'clinical_ner_tags'``, ``'clinical_ner_chunks_re'``, ``'document'``, ``'clinical_ner_chunks'``, ``'assertion'``, ``'clinical_relations'``, ``'tokens'``, ``'embeddings'``, ``'pos_tags'``, ``'dependencies'``. Here is the result of `clinical_ner_chunks` : ```bash | # | chunks | begin | end | entities | |----|-------------------------------|-------|-----|---------------| | 0 | admitted | 7 | 14 | OCCURRENCE | | 1 | The John Hopkins Hospital | 19 | 43 | CLINICAL_DEPT | | 2 | 2 days ago | 45 | 54 | DATE | | 3 | gestational diabetes mellitus | 74 | 102 | PROBLEM | | 4 | diagnosed | 104 | 112 | OCCURRENCE | | 5 | denied | 119 | 124 | EVIDENTIAL | | 6 | pain | 126 | 129 | PROBLEM | | 7 | any headache | 135 | 146 | PROBLEM | | 8 | seen | 157 | 160 | OCCURRENCE | | 9 | the endocrinology service | 165 | 189 | CLINICAL_DEPT | | 10 | discharged | 203 | 212 | OCCURRENCE | | 11 | 03/02/2018 | 217 | 226 | DATE | | 12 | insulin glargine | 243 | 258 | TREATMENT | | 13 | insulin lispro | 274 | 287 | TREATMENT | | 14 | metformin | 294 | 302 | TREATMENT | | 15 | two times a day | 312 | 326 | FREQUENCY | | 16 | close follow-up | 337 | 351 | OCCURRENCE | | 17 | endocrinology | 358 | 370 | CLINICAL_DEPT | | 18 | discharge | 377 | 385 | OCCURRENCE | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_clinical_doc_era| |Type:|pipeline| |Compatibility:|Healthcare NLP 2.6.0 +| |License:|Licensed| |Edition:|Official| |Language:|[en]| {:.h2_title} ## Included Models - ``ner_clinical_events`` - ``assertion_dl`` - ``re_temporal_events_clinical`` --- layout: model title: Detect Restaurant-related Terminology author: John Snow Labs name: nerdl_restaurant_100d date: 2021-12-31 tags: [ner, restaurant, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.3.4 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with `glove_100d` embeddings to detect restaurant-related terminology. ## Predicted Entities `Location`, `Cuisine`, `Amenity`, `Restaurant_Name`, `Dish`, `Rating`, `Hours`, `Price` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_RESTAURANT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_RESTAURANT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_restaurant_100d_en_3.3.4_3.0_1640949258750.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_restaurant_100d_en_3.3.4_3.0_1640949258750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_100d") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nerdl = NerDLModel.pretrained("nerdl_restaurant_100d")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, nerdl, ner_converter]) text = """Hong Kong’s favourite pasta bar also offers one of the most reasonably priced lunch sets in town! With locations spread out all over the territory Sha Tin – Pici’s formidable lunch menu reads like a highlight reel of the restaurant. Choose from starters like the burrata and arugula salad or freshly tossed tuna tartare, and reliable handmade pasta dishes like pappardelle. Finally, round out your effortless Italian meal with a tidy one-pot tiramisu, of course, an espresso to power you through the rest of the day.""" data = spark.createDataFrame([[text]]).toDF("text") result = nlp_pipeline.fit(data).transform(data) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_100d") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val nerdl = NerDLModel.pretrained("nerdl_restaurant_100d") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, nerdl, ner_converter)) val data = Seq("Hong Kong’s favourite pasta bar also offers one of the most reasonably priced lunch sets in town! With locations spread out all over the territory Sha Tin – Pici’s formidable lunch menu reads like a highlight reel of the restaurant. Choose from starters like the burrata and arugula salad or freshly tossed tuna tartare, and reliable handmade pasta dishes like pappardelle. Finally, round out your effortless Italian meal with a tidy one-pot tiramisu, of course, an espresso to power you through the rest of the day.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.restaurant").predict("""Hong Kong’s favourite pasta bar also offers one of the most reasonably priced lunch sets in town! With locations spread out all over the territory Sha Tin – Pici’s formidable lunch menu reads like a highlight reel of the restaurant. Choose from starters like the burrata and arugula salad or freshly tossed tuna tartare, and reliable handmade pasta dishes like pappardelle. Finally, round out your effortless Italian meal with a tidy one-pot tiramisu, of course, an espresso to power you through the rest of the day.""") ```

## Results ```bash +---------------------------+---------------+ |chunk |ner_label | +---------------------------+---------------+ |favourite |Rating | |pasta bar |Dish | |most reasonably |Price | |lunch |Hours | |in town! |Location | |Sha Tin – Pici’s |Restaurant_Name| |burrata |Dish | |arugula salad |Dish | |freshly tossed tuna tartare|Dish | |reliable |Price | |handmade pasta |Dish | |pappardelle |Dish | |effortless |Amenity | |Italian |Cuisine | |tidy one-pot |Amenity | |espresso |Dish | +---------------------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_restaurant_100d| |Type:|ner| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|14.2 MB| ## Data Source [https://groups.csail.mit.edu/sls/downloads/restaurant/](https://groups.csail.mit.edu/sls/downloads/restaurant/) ## Benchmarking ```bash label precision recall f1-score support B-Amenity 0.77 0.75 0.76 545 B-Cuisine 0.86 0.88 0.87 524 B-Dish 0.84 0.80 0.82 303 B-Hours 0.67 0.72 0.69 197 B-Location 0.89 0.89 0.89 807 B-Price 0.86 0.87 0.86 169 B-Rating 0.87 0.79 0.83 221 B-Restaurant_Name 0.91 0.94 0.92 388 I-Amenity 0.80 0.75 0.77 561 I-Cuisine 0.71 0.71 0.71 135 I-Dish 0.66 0.77 0.71 104 I-Hours 0.87 0.84 0.86 306 I-Location 0.92 0.87 0.89 834 I-Price 0.64 0.81 0.71 52 I-Rating 0.80 0.85 0.82 118 I-Restaurant_Name 0.82 0.89 0.85 359 O 0.95 0.96 0.96 8634 accuracy - - 0.91 14257 macro-avg 0.81 0.83 0.82 14257 weighted-avg 0.91 0.91 0.91 14257 ``` --- layout: model title: Chinese XLMRoBerta Embeddings (from coastalcph) author: John Snow Labs name: xlmroberta_embeddings_fairlex_cail_minilm date: 2022-05-13 tags: [zh, open_source, xlm_roberta, embeddings, fairlex] task: Embeddings language: zh edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `fairlex-cail-minilm` is a Chinese model orginally trained by `coastalcph`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_fairlex_cail_minilm_zh_3.4.4_3.0_1652439739577.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_fairlex_cail_minilm_zh_3.4.4_3.0_1652439739577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_fairlex_cail_minilm| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|403.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/coastalcph/fairlex-cail-minilm - https://coastalcph.github.io - https://github.com/iliaschalkidis - https://twitter.com/KiddoThe2B --- layout: model title: XLNet Embeddings (Large) author: John Snow Labs name: xlnet_large_cased date: 2020-04-28 task: Embeddings language: en nav_key: models edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: XlnetEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description XLNet is a new unsupervised language representation learning method based on a novel generalized permutation language modeling objective. Additionally, XLNet employs Transformer-XL as the backbone model, exhibiting excellent performance for language tasks involving long context. Overall, XLNet achieves state-of-the-art (SOTA) results on various downstream language tasks including question answering, natural language inference, sentiment analysis, and document ranking. The details are described in the paper "[XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlnet_large_cased_en_2.5.0_2.4_1588074397954.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlnet_large_cased_en_2.5.0_2.4_1588074397954.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = XlnetEmbeddings.pretrained("xlnet_large_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = XlnetEmbeddings.pretrained("xlnet_large_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.xlnet_large_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_xlnet_large_cased_embeddings I [0.9742076396942139, -0.6181889772415161, 0.45... love [-0.7322277426719666, -1.7254987955093384, -0.... NLP [1.6873085498809814, -0.8617655038833618, 0.46... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlnet_large_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.5.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/zihangdai/xlnet](https://github.com/zihangdai/xlnet) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from smonah) author: John Snow Labs name: distilbert_qa_smonah_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `smonah`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_smonah_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772837756.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_smonah_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772837756.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_smonah_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_smonah_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_smonah_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/smonah/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering Cased model (from KFlash) author: John Snow Labs name: bert_qa_kflash_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `KFlash`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_kflash_finetuned_squad_en_4.0.0_3.0_1657186070833.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_kflash_finetuned_squad_en_4.0.0_3.0_1657186070833.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_kflash_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_kflash_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_kflash_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KFlash/bert-finetuned-squad --- layout: model title: Korean T5ForConditionalGeneration Base Cased model (from KETI-AIR) author: John Snow Labs name: t5_ke_base date: 2023-01-30 tags: [ko, open_source, t5, tensorflow] task: Text Generation language: ko edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ke-t5-base-ko` is a Korean model originally trained by `KETI-AIR`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_ke_base_ko_4.3.0_3.0_1675104551769.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_ke_base_ko_4.3.0_3.0_1675104551769.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ke_base","ko") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ke_base","ko") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_lijingxin_generic_model_2| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/lijingxin/dummy-model-2 --- layout: model title: Legal Effectiveness Clause Binary Classifier author: John Snow Labs name: legclf_effectiveness_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `effectiveness` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `effectiveness` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_effectiveness_clause_en_1.0.0_3.2_1660123466311.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_effectiveness_clause_en_1.0.0_3.2_1660123466311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_effectiveness_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[effectiveness]| |[other]| |[other]| |[effectiveness]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_effectiveness_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support effectiveness 0.94 0.86 0.90 37 other 0.95 0.98 0.96 93 accuracy - - 0.95 130 macro-avg 0.94 0.92 0.93 130 weighted-avg 0.95 0.95 0.95 130 ``` --- layout: model title: Mapping Companies IRS to Edgar Database author: John Snow Labs name: finmapper_edgar_irs date: 2022-08-18 tags: [en, finance, companies, edgar, data, augmentation, irs, licensed] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Chunk Mapper model allows you to, given a detected IRS with any NER model, augment it with information available in the SEC Edgar database. Some of the fields included in this Chunk Mapper are: - Company Name - Sector - Former names - Address, Phone, State - Dates where the company submitted filings - etc. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FIN_LEG_COMPANY_AUGMENTATION/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_edgar_irs_en_1.0.0_3.2_1660817662889.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_edgar_irs_en_1.0.0_3.2_1660817662889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash [Row(mappings=[Row(annotatorType='labeled_dependency', begin=0, end=8, result='Masterworks 096, LLC', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='RETAIL-RETAIL STORES, NEC [5990]', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'sic', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='5990', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'sic_code', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='873474341', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'irs_number', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='1231', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'fiscal_year_end', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='NY', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'state_location', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='DE', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'state_incorporation', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='225 LIBERTY STREET', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'business_street', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='NEW YORK', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'business_city', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='NY', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'business_state', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='10281', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'business_zip', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='2035185172', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'business_phone', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'former_name', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'former_name_date', 'all_relations': ''}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='2022-01-10', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'date', 'all_relations': '2022-04-26:::2021-11-17'}, embeddings=[]), Row(annotatorType='labeled_dependency', begin=0, end=8, result='1894064', metadata={'sentence': '0', 'chunk': '0', 'entity': '873474341', 'relation': 'company_id', 'all_relations': ''}, embeddings=[])])] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_edgar_irs| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|5.7 MB| ## References Manually scrapped Edgar Database --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Evelyn18) author: John Snow Labs name: distilbert_qa_base_uncased_becas_6 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-becas-6` is a English model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_6_en_4.3.0_3.0_1672767591313.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_becas_6_en_4.3.0_3.0_1672767591313.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_6","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_becas_6","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_becas_6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Evelyn18/distilbert-base-uncased-becas-6 --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from tli8hf) author: John Snow Labs name: roberta_qa_unqover_base_news date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unqover-roberta-base-newsqa` is a English model originally trained by `tli8hf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_base_news_en_4.3.0_3.0_1674224491216.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_base_news_en_4.3.0_3.0_1674224491216.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_base_news","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_base_news","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_unqover_base_news| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|463.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/tli8hf/unqover-roberta-base-newsqa --- layout: model title: Spanish BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488 date: 2022-06-03 tags: [es, open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-cased-finetuned-spa-squad2-es` is a Spanish model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488_es_4.0.0_3.0_1654249712848.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488_es_4.0.0_3.0_1654249712848.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2.bert.base_cased.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_cased_finetuned_spa_squad2_es_mrm8488| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-base-spanish-wwm-cased-finetuned-spa-squad2-es - https://github.com/dccuchile/beto - https://github.com/google-research/bert/blob/master/multilingual.md - https://github.com/ccasimiro88/TranslateAlignRetrieve - https://twitter.com/mrm8488 - https://github.com/google-research/bert - https://github.com/josecannete/spanish-corpora - https://github.com/dccuchile/beto/blob/master/README.md --- layout: model title: Finnish RobertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: roberta_qa_ADDI_FI_RoBERTa date: 2022-06-20 tags: [open_source, question_answering, roberta] task: Question Answering language: fi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-FI-RoBERTa` is a Finnish model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FI_RoBERTa_fi_4.0.0_3.0_1655726349369.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ADDI_FI_RoBERTa_fi_4.0.0_3.0_1655726349369.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ADDI_FI_RoBERTa","fi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ADDI_FI_RoBERTa","fi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fi.answer_question.roberta.fi_tuned.by_Gantenbein").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ADDI_FI_RoBERTa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fi| |Size:|421.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-FI-RoBERTa --- layout: model title: Chinese Part of Speech Tagger(Large, UPOS, Chinese Wikipedia Texts author: John Snow Labs name: bert_pos_chinese_roberta_large_upos date: 2022-04-26 tags: [bert, pos, part_of_speech, zh, open_source] task: Part of Speech Tagging language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-roberta-large-upos` is a Chinese model orginally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_chinese_roberta_large_upos_zh_3.4.2_3.0_1650993118594.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_chinese_roberta_large_upos_zh_3.4.2_3.0_1650993118594.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_chinese_roberta_large_upos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/KoichiYasuoka/chinese-roberta-large-upos - https://universaldependencies.org/u/pos/ - https://github.com/KoichiYasuoka/esupar --- layout: model title: Arabic BertForMaskedLM Base Cased model (from aubmindlab) author: John Snow Labs name: bert_embeddings_base_arabertv01 date: 2022-12-02 tags: [ar, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: ar edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-arabertv01` is a Arabic model originally trained by `aubmindlab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv01_ar_4.2.4_3.0_1670015781720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_arabertv01_ar_4.2.4_3.0_1670015781720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv01","ar") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_arabertv01","ar") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_arabertv01| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|507.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/aubmindlab/bert-base-arabertv01 - https://github.com/google-research/bert - https://arxiv.org/abs/2003.00104 - https://github.com/WissamAntoun/pydata_khobar_meetup - http://alt.qcri.org/farasa/segmenter.html - /aubmindlab/bert-base-arabertv01/blob/main/(https://github.com/google-research/bert/blob/master/multilingual.md) - https://github.com/elnagara/HARD-Arabic-Dataset - https://www.aclweb.org/anthology/D15-1299 - https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf - https://github.com/mohamedadaly/LABR - http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp - https://github.com/husseinmozannar/SOQAL - https://github.com/aub-mind/arabert/blob/master/AraBERT/README.md - https://arxiv.org/abs/2003.00104v2 - https://archive.org/details/arwiki-20190201 - https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4 - https://www.aclweb.org/anthology/W19-4619 - https://sites.aub.edu.lb/mindlab/ - https://www.yakshof.com/#/ - https://www.behance.net/rahalhabib - https://www.linkedin.com/in/wissam-antoun-622142b4/ - https://twitter.com/wissam_antoun - https://github.com/WissamAntoun - https://www.linkedin.com/in/fadybaly/ - https://twitter.com/fadybaly - https://github.com/fadybaly --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from jamesmarcel) author: John Snow Labs name: xlmroberta_ner_jamesmarcel_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `jamesmarcel`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jamesmarcel_base_finetuned_panx_de_4.1.0_3.0_1660434088567.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jamesmarcel_base_finetuned_panx_de_4.1.0_3.0_1660434088567.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jamesmarcel_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jamesmarcel_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jamesmarcel_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jamesmarcel/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Stop Words Cleaner for Hebrew author: John Snow Labs name: stopwords_he date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: he edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, he] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_he_he_2.5.4_2.4_1594742441877.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_he_he_2.5.4_2.4_1594742441877.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_he", "he") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("מלבד היותו מלך הצפון, ג'ון סנואו הוא רופא אנגלי ומוביל בפיתוח הרדמה והיגיינה רפואית.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_he", "he") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("מלבד היותו מלך הצפון, ג'ון סנואו הוא רופא אנגלי ומוביל בפיתוח הרדמה והיגיינה רפואית.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""מלבד היותו מלך הצפון, ג'ון סנואו הוא רופא אנגלי ומוביל בפיתוח הרדמה והיגיינה רפואית"""] stopword_df = nlu.load('he.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=5, end=9, result='היותו', metadata={'sentence': '0'}), Row(annotatorType='token', begin=11, end=13, result='מלך', metadata={'sentence': '0'}), Row(annotatorType='token', begin=15, end=19, result='הצפון', metadata={'sentence': '0'}), Row(annotatorType='token', begin=20, end=20, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=22, end=25, result="ג'ון", metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_he| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|he| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English LongformerForQuestionAnswering model (from valhalla) author: John Snow Labs name: longformer_base_base_qa_squad2 date: 2022-06-15 tags: [open_source, longformer, question_answering, en] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `longformer-base-4096-finetuned-squadv1` is a English model originally trained by `valhalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_base_base_qa_squad2_en_4.0.0_3.0_1655292497074.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_base_base_qa_squad2_en_4.0.0_3.0_1655292497074.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_base_base_qa_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_base_base_qa_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.longformer.base").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_base_base_qa_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|550.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References https://huggingface.co/valhalla/longformer-base-4096-finetuned-squadv1 --- layout: model title: Extract Cancer Therapies and Posology Information author: John Snow Labs name: ner_oncology_unspecific_posology_wip date: 2022-09-30 tags: [licensed, clinical, oncology, en, ner, treatment, posology] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of treatments and posology information using general labels (low granularity). Definitions of Predicted Entities: - `Cancer_Therapy`: Mentions of cancer treatments, including chemotherapy, radiotherapy, surgery and other. - `Posology_Information`: Terms related to the posology of the treatment, including duration, frequencies and dosage. ## Predicted Entities `Cancer_Therapy`, `Posology_Information` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_wip_en_4.0.0_3.0_1664563752336.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_wip_en_4.0.0_3.0_1664563752336.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:---------------------| | adriamycin | Cancer_Therapy | | cyclophosphamide | Cancer_Therapy | | 600 mg/m2 | Posology_Information | | over six courses | Posology_Information | | second cycle | Posology_Information | | chemotherapy | Cancer_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_unspecific_posology_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|841.3 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Posology_Information 1908.0 86.0 439.0 2347.0 0.96 0.81 0.88 Cancer_Therapy 1685.0 94.0 440.0 2125.0 0.95 0.79 0.86 macro_avg 3593.0 180.0 879.0 4472.0 0.95 0.80 0.87 micro_avg NaN NaN NaN NaN 0.95 0.80 0.87 ``` --- layout: model title: English RobertaForQuestionAnswering (from eAsyle) author: John Snow Labs name: roberta_qa_testABSA date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `testABSA` is a English model originally trained by `eAsyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_testABSA_en_4.0.0_3.0_1655739870848.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_testABSA_en_4.0.0_3.0_1655739870848.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_testABSA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_testABSA","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.testabsa.by_eAsyle").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_testABSA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|426.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/eAsyle/testABSA --- layout: model title: Indonesian T5ForConditionalGeneration Small Cased model (from panggi) author: John Snow Labs name: t5_small_summarization_cased date: 2023-01-31 tags: [id, open_source, t5, tensorflow] task: Text Generation language: id edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-indonesian-summarization-cased` is a Indonesian model originally trained by `panggi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_summarization_cased_id_4.3.0_3.0_1675126407873.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_summarization_cased_id_4.3.0_3.0_1675126407873.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_summarization_cased","id") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_summarization_cased","id") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_summarization_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|id| |Size:|287.9 MB| ## References - https://huggingface.co/panggi/t5-small-indonesian-summarization-cased - https://github.com/kata-ai/indosum --- layout: model title: Finnish asr_wav2vec2_large_uralic_voxpopuli_v2_finnish TFWav2Vec2ForCTC from Finnish-NLP author: John Snow Labs name: asr_wav2vec2_large_uralic_voxpopuli_v2_finnish date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_uralic_voxpopuli_v2_finnish` is a Finnish model originally trained by Finnish-NLP. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_fi_4.2.0_3.0_1664037968087.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_uralic_voxpopuli_v2_finnish_fi_4.2.0_3.0_1664037968087.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_uralic_voxpopuli_v2_finnish", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_uralic_voxpopuli_v2_finnish", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_uralic_voxpopuli_v2_finnish| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: Legal Management contract Document Classifier (Longformer) author: John Snow Labs name: legclf_management_contract date: 2022-10-24 tags: [en, legal, classification, document, agreement, contract, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_management_contract` model is a Legal Longformer Document Classifier to classify if the document belongs to the class management-contract or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `other`, `management-contract` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_management_contract_en_1.0.0_3.0_1666620999192.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_management_contract_en_1.0.0_3.0_1666620999192.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token") embeddings = nlp.LongformerEmbeddings.pretrained("legal_longformer_base", "en")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") sembeddings = nlp.SentenceEmbeddings()\ .setInputCols(["document", "embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_management_contract", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, sembeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[management-contract]| |[other]| |[other]| |[management-contract]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_management_contract| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support management-contract 0.97 0.94 0.95 31 other 0.96 0.98 0.97 52 accuracy - - 0.96 83 macro-avg 0.96 0.96 0.96 83 weighted-avg 0.96 0.96 0.96 83 ``` --- layout: model title: Fast Neural Machine Translation Model from Argentine Sign Language to Spanish author: John Snow Labs name: opus_mt_aed_es date: 2021-06-01 tags: [open_source, seq2seq, translation, aed, es, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: aed target languages: es {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_aed_es_xx_3.1.0_2.4_1622560123437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_aed_es_xx_3.1.0_2.4_1622560123437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_aed_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Chinese Bert Embeddings (Large) author: John Snow Labs name: bert_embeddings_uer_large date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `uer_large` is a Chinese model orginally trained by `junnyu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_uer_large_zh_3.4.2_3.0_1649669433036.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_uer_large_zh_3.4.2_3.0_1649669433036.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_uer_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| ## References - https://huggingface.co/junnyu/uer_large - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://share.weiyun.com/5G90sMJ --- layout: model title: Legal Severance Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_severance_agreement date: 2022-12-06 tags: [en, legal, classification, agreement, severance, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_severance_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `severance-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `severance-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_severance_agreement_en_1.0.0_3.0_1670358097286.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_severance_agreement_en_1.0.0_3.0_1670358097286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[severance-agreement]| |[other]| |[other]| |[severance-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_severance_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.99 1.00 111 severance-agreement 0.98 1.00 0.99 57 accuracy - - 0.99 168 macro-avg 0.99 1.00 0.99 168 weighted-avg 0.99 0.99 0.99 168 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from manishiitg) author: John Snow Labs name: roberta_qa_distilrobert_base_squadv2_328seq_128stride_test date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilrobert-base-squadv2-328seq-128stride-test` is a English model originally trained by `manishiitg`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_distilrobert_base_squadv2_328seq_128stride_test_en_4.3.0_3.0_1674210316686.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_distilrobert_base_squadv2_328seq_128stride_test_en_4.3.0_3.0_1674210316686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilrobert_base_squadv2_328seq_128stride_test","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilrobert_base_squadv2_328seq_128stride_test","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilrobert_base_squadv2_328seq_128stride_test| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/manishiitg/distilrobert-base-squadv2-328seq-128stride-test --- layout: model title: English RobertaForQuestionAnswering (from mrm8488) author: John Snow Labs name: roberta_qa_roberta_base_1B_1_finetuned_squadv2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-1B-1-finetuned-squadv2` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_1B_1_finetuned_squadv2_en_4.0.0_3.0_1655729733962.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_1B_1_finetuned_squadv2_en_4.0.0_3.0_1655729733962.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_1B_1_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_1B_1_finetuned_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_1B_1_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|447.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/roberta-base-1B-1-finetuned-squadv2 - https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: RoBERTa large model author: John Snow Labs name: roberta_large date: 2021-05-20 tags: [en, english, embeddings, roberta, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in [this paper](https://arxiv.org/abs/1907.11692) and first released in [this repository](https://github.com/pytorch/fairseq/tree/master/examples/roberta). This model is case-sensitive: it makes a difference between english and English. RoBERTa is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it was pretrained on the raw texts only, with no humans labeling them in any way (which is why it can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model randomly masks 15% of the words in the input then runs the entire masked sentence through the model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the sentence. This way, the model learns an inner representation of the English language that can then be used to extract features useful for downstream tasks: if you have a dataset of labeled sentences, for instance, you can train a standard classifier using the features produced by the BERT model as inputs. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_large_en_3.1.0_2.4_1621523610703.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_large_en_3.1.0_2.4_1621523610703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = RoBertaEmbeddings.pretrained("roberta_large", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ``` ```scala val embeddings = RoBertaEmbeddings.pretrained("roberta_large", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.roberta.large").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_large| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|en| |Case sensitive:|true| ## Data Source [https://huggingface.co/roberta-large](https://huggingface.co/roberta-large) ## Benchmarking ```bash When fine-tuned on downstream tasks, this model achieves the following results: Glue test results: | Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | |:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| | | 90.2 | 92.2 | 94.7 | 96.4 | 68.0 | 96.4 | 90.9 | 86.6 | ``` --- layout: model title: Welsh Lemmatizer author: John Snow Labs name: lemma date: 2021-04-02 tags: [cy, open_source, lemmatizer] task: Lemmatization language: cy edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a dictionary-based lemmatizer that assigns all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_cy_3.0.0_3.0_1617389338320.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_cy_3.0.0_3.0_1617389338320.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------+ | lemma| +------------+ | Dywedir| | yn| | aml| | taw| | rygbi| | '| | r| | undeb| | bod| | mabolgamp| |cenedlaethol| | Cymru| | ,| | er| | bod| | pêl-droed| | yn| | denu| | mawr| | o| +------------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|cy| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) version 2.7. ## Benchmarking ```bash Precision=0.74, Recall=0.71, F1-score=0.72 ``` --- layout: model title: Stopwords Remover for Hebrew language (226 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, he, open_source] task: Stop Words Removal language: he edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_he_3.4.1_3.0_1646673023597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_he_3.4.1_3.0_1646673023597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+ |result | +-----------+ |[טוב, ממני]| +-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|he| |Size:|2.1 KB| --- layout: model title: Romanian T5ForConditionalGeneration Small Cased model (from BlackKakapo) author: John Snow Labs name: t5_small_grammar date: 2023-01-31 tags: [ro, open_source, t5, tensorflow] task: Text Generation language: ro edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-grammar-ro` is a Romanian model originally trained by `BlackKakapo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_grammar_ro_4.3.0_3.0_1675126317557.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_grammar_ro_4.3.0_3.0_1675126317557.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_grammar","ro") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_grammar","ro") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_grammar| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ro| |Size:|284.0 MB| ## References - https://huggingface.co/BlackKakapo/t5-small-grammar-ro - https://img.shields.io/badge/V.1-03.08.2022-brightgreen --- layout: model title: Recognize Entities OntoNotes - BERT Medium author: John Snow Labs name: onto_recognize_entities_bert_medium date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, en, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `small_bert_L8_512` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_medium_en_2.7.0_2.4_1607510751761.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_medium_en_2.7.0_2.4_1607510751761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_bert_medium') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_medium") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu text = ["""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament."""] ner_df = nlu.load('en.ner.onto.bert.medium').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_medium| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: Tswana RobertaForMaskedLM Cased model (from MoseliMotsoehli) author: John Snow Labs name: roberta_embeddings_tswanabert date: 2022-12-12 tags: [tn, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: tn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `TswanaBert` is a Tswana model originally trained by `MoseliMotsoehli`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_tswanabert_tn_4.2.4_3.0_1670858564493.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_tswanabert_tn_4.2.4_3.0_1670858564493.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_tswanabert","tn") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_tswanabert","tn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_tswanabert| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tn| |Size:|231.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/MoseliMotsoehli/TswanaBert - https://wortschatz.uni-leipzig.de/en/download - http://doi.org/10.5281/zenodo.3668495 - http://setswana.blogspot.com/ - https://omniglot.com/writing/tswana.php - http://www.dailynews.gov.bw/ - http://www.mmegi.bw/index.php - https://tsena.co.bw - http://www.botswana.co.za/Cultural_Issues-travel/botswana-country-guide-en-route.html - https://www.poemhunter.com/poem/2013-setswana/ - https://www.poemhunter.com/poem/ngwana-wa-mosetsana/ --- layout: model title: English image_classifier_vit_exper2_mesum5 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_exper2_mesum5 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_exper2_mesum5` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper2_mesum5_en_4.1.0_3.0_1660167892531.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_exper2_mesum5_en_4.1.0_3.0_1660167892531.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_exper2_mesum5", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_exper2_mesum5", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_exper2_mesum5| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: Stopwords Remover for Latvian language (161 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, lv, open_source] task: Stop Words Removal language: lv edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_lv_3.4.1_3.0_1646673201505.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_lv_3.4.1_3.0_1646673201505.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+ |result | +---------------------------+ |[Jūs, neesat, labāks, mani]| +---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|lv| |Size:|1.8 KB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from zshang3) author: John Snow Labs name: distilbert_qa_zshang3_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `zshang3`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_zshang3_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773401928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_zshang3_base_uncased_finetuned_squad_en_4.3.0_3.0_1672773401928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zshang3_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_zshang3_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_zshang3_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/zshang3/distilbert-base-uncased-finetuned-squad --- layout: model title: Detect Biomedical Entities in English author: gokhanturer name: ner_protein_glove date: 2022-02-20 tags: [open_source, ner, glove_100d, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.1.2 spark_version: 3.0 supported: false annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity Recognition model that finds `Protein` entitites in biomedical texts. ## Predicted Entities `Protein` {:.btn-box} [Open in Colab](https://colab.research.google.com/drive/1npHXVQbqZ5rFOTReG2DjOGuQFR3cX34Q#scrollTo=Lq8fqJfmFY9V){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/gokhanturer/ner_protein_glove_en_3.1.2_3.0_1645385210378.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/gokhanturer/ner_protein_glove_en_3.1.2_3.0_1645385210378.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence = SentenceDetector()\ .setInputCols(['document'])\ .setOutputCol('sentence') token = Tokenizer()\ .setInputCols(['sentence'])\ .setOutputCol('token') glove_embeddings = WordEmbeddingsModel.pretrained()\ .setInputCols(["document", "token"])\ .setOutputCol("embeddings") loaded_ner_model = NerDLModel.pretrained("ner_protein_glove", "en")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_span") ner_prediction_pipeline = Pipeline(stages = [ document, sentence, token, glove_embeddings, loaded_ner_model, converter ]) text = ''' MACROPHAGES ARE MONONUCLEAR phagocytes that reside within almost all tissues including adipose tissue, where they are identifiable as distinct populations with tissue-specific morphology, localization, and function (1). During the process of atherosclerosis, monocytes adhere to the endothelium and migrate into the intima, express scavenger receptors, and bind internalized lipoprotein particles resulting in the formation of foam cells (2). In obesity, adipose tissue contains an increased number of resident macrophages (3, 4). Macrophage accumulation in proportion to adipocyte size may increase the adipose tissue production of proinflammatory and acute-phase molecules and thereby contribute to the pathophysiological consequences of obesity (1, 3). These facts indicate that macrophages play an important role in a variety of diseases. When activated, macrophages release stereotypical profiles of cytokines and biological molecules such as nitric oxide TNF-α, IL-6, and IL-1 (5). TNF-α is a potent chemoattractant (6) and originates predominantly from residing mouse peritoneal macrophages (MPM) and mast cells (7). TNF-α induces leukocyte adhesion and degranulation, stimulates nicotinamide adenine dinucleotide phosphate (NADPH) oxidase, and enhances expression of IL-2 receptors and expression of E-selectin and intercellular adhesion molecules on the endothelium (8). TNF-α also stimulates expression of IL-1, IL-2, IL-6, and platelet-activating factor receptor (9). In addition, TNF-α decreases insulin sensitivity and increases lipolysis in adipocytes (10, 11). IL-6 also increase lipolysis and has been implicated in the hypertriglyceridemia and increased serum free fatty acid levels associated with obesity (12). Increased IL-6 signaling induces the expression of C-reactive protein and haptoglubin in liver (13). Recombinant IL-6 treatment increases atherosclerotic lesion size 5-fold (14). IL-6 also dose-dependently increases macrophage oxidative low-density lipoprotein (LDL) degradation and CD36 mRNA expression in vitro (15). These data clearly indicate that IL-6 and TNF-α are important pathogenetic factors associated with obesity, insulin resistance, and atherosclerosis. However, the factors regulating gene expression of these cytokines in macrophages have not been fully clarified. ''' sample_data = spark.createDataFrame([[text]]).toDF("text") prediction_model = ner_prediction_pipeline.fit(sample_data) preds = prediction_model.transform(sample_data) ``` ```scala val document = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val token = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val glove_embeddings = WordEmbeddingsModel.pretrained() .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val loaded_ner_model = NerDLModel.pretrained("ner_protein_glove", "en") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_span") val ner_prediction_pipeline = new Pipeline().setStages(Array( document, sentence, token, glove_embeddings, loaded_ner_model, converter )) val text =Seq(" MACROPHAGES ARE MONONUCLEAR phagocytes that reside within almost all tissues including adipose tissue, where they are identifiable as distinct populations with tissue-specific morphology, localization, and function (1). During the process of atherosclerosis, monocytes adhere to the endothelium and migrate into the intima, express scavenger receptors, and bind internalized lipoprotein particles resulting in the formation of foam cells (2). In obesity, adipose tissue contains an increased number of resident macrophages (3, 4). Macrophage accumulation in proportion to adipocyte size may increase the adipose tissue production of proinflammatory and acute-phase molecules and thereby contribute to the pathophysiological consequences of obesity (1, 3). These facts indicate that macrophages play an important role in a variety of diseases. When activated, macrophages release stereotypical profiles of cytokines and biological molecules such as nitric oxide TNF-α, IL-6, and IL-1 (5). TNF-α is a potent chemoattractant (6) and originates predominantly from residing mouse peritoneal macrophages (MPM) and mast cells (7). TNF-α induces leukocyte adhesion and degranulation, stimulates nicotinamide adenine dinucleotide phosphate (NADPH) oxidase, and enhances expression of IL-2 receptors and expression of E-selectin and intercellular adhesion molecules on the endothelium (8). TNF-α also stimulates expression of IL-1, IL-2, IL-6, and platelet-activating factor receptor (9). In addition, TNF-α decreases insulin sensitivity and increases lipolysis in adipocytes (10, 11). IL-6 also increase lipolysis and has been implicated in the hypertriglyceridemia and increased serum free fatty acid levels associated with obesity (12). Increased IL-6 signaling induces the expression of C-reactive protein and haptoglubin in liver (13). Recombinant IL-6 treatment increases atherosclerotic lesion size 5-fold (14). IL-6 also dose-dependently increases macrophage oxidative low-density lipoprotein (LDL) degradation and CD36 mRNA expression in vitro (15). These data clearly indicate that IL-6 and TNF-α are important pathogenetic factors associated with obesity, insulin resistance, and atherosclerosis. However, the factors regulating gene expression of these cytokines in macrophages have not been fully clarified. ").toDF("text") val preds = prediction_model.fit(sample_data).transform(sample_data) ```

## Results ```bash +--------------+-------+ |chunk |entity | +--------------+-------+ |IL-6 |Protein| |IL-1 (5). |Protein| |TNF-α |Protein| |IL-2 receptors|Protein| |IL-2 |Protein| |IL-6 |Protein| |insulin |Protein| |haptoglubin |Protein| |CD36 |Protein| |IL-6 |Protein| |insulin |Protein| +--------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|Ner_glove_100d| |Type:|ner| |Compatibility:|Spark NLP 3.1.2+| |License:|Open Source| |Edition:|Community| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|14.3 MB| |Dependencies:|glove100d| ## References This model trained based on following [dataset](https://github.com/gokhanturer/NER_Model_SparkNLP/blob/main/BioNLP09_IOB_train.conll) ## Benchmarking ```bash label precision recall f1-score support B-Protein 0.86 0.87 0.87 3589 I-Protein 0.84 0.86 0.85 4078 O 0.99 0.98 0.98 66957 accuracy - - 0.97 74624 macro-avg 0.90 0.91 0.90 74624 weighted-avg 0.97 0.97 0.97 74624 ``` --- layout: model title: Fast Neural Machine Translation Model from Lingala to English author: John Snow Labs name: opus_mt_ln_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ln, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ln` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ln_en_xx_2.7.0_2.4_1609170591201.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ln_en_xx_2.7.0_2.4_1609170591201.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ln_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate English to Niuean Pipeline author: John Snow Labs name: translate_en_niu date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, niu, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `niu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_niu_xx_2.7.0_2.4_1609686690157.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_niu_xx_2.7.0_2.4_1609686690157.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_niu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_niu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.niu').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_niu| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish BertForQuestionAnswering Base Uncased model (from stevemobs) author: John Snow Labs name: bert_qa_base_spanish_wwm_uncased_finetuned_squad date: 2022-07-07 tags: [es, open_source, bert, question_answering] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-uncased-finetuned-squad_es` is a Spanish model originally trained by `stevemobs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_spanish_wwm_uncased_finetuned_squad_es_4.0.0_3.0_1657183450072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_spanish_wwm_uncased_finetuned_squad_es_4.0.0_3.0_1657183450072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_spanish_wwm_uncased_finetuned_squad","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_spanish_wwm_uncased_finetuned_squad","es") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("¿Cuál es mi nombre?", "Mi nombre es Clara y vivo en Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_spanish_wwm_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|410.3 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/stevemobs/bert-base-spanish-wwm-uncased-finetuned-squad_es --- layout: model title: Spanish BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_es_cased date: 2022-12-02 tags: [es, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-es-cased` is a Spanish model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_es_cased_es_4.2.4_3.0_1670017490926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_es_cased_es_4.2.4_3.0_1670017490926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_es_cased","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_es_cased","es") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_es_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|399.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-es-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate English to Brazilian Sign Language Pipeline author: John Snow Labs name: translate_en_bzs date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, bzs, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `bzs` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_bzs_xx_2.7.0_2.4_1609689757673.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_bzs_xx_2.7.0_2.4_1609689757673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_bzs", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_bzs", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.bzs').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_bzs| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Part of Speech for Hindi author: John Snow Labs name: pos_ud_hdtb date: 2021-03-09 tags: [part_of_speech, open_source, hindi, pos_ud_hdtb, hi] task: Part of Speech Tagging language: hi edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - DET - PROPN - ADP - ADV - ADJ - NOUN - NUM - AUX - PUNCT - PRON - VERB - CCONJ - PART - SCONJ - X - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_hdtb_hi_3.0.0_3.0_1615292181587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_hdtb_hi_3.0.0_3.0_1615292181587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_hdtb", "hi") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['जॉन स्नो लैब्स से नमस्ते! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_hdtb", "hi") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("जॉन स्नो लैब्स से नमस्ते! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""जॉन स्नो लैब्स से नमस्ते! ""] token_df = nlu.load('hi.pos').predict(text) token_df ```

## Results ```bash token pos 0 जॉन PROPN 1 स्नो PROPN 2 लैब्स PROPN 3 से ADP 4 नमस्ते NOUN 5 ! VERB ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_hdtb| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|hi| --- layout: model title: Pipeline to Summarize Clinical Notes (Augmented) author: John Snow Labs name: summarizer_clinical_jsl_augmented_pipeline date: 2023-05-29 tags: [licensed, en, clinical, text_summarization, augmented] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_jsl_augmented](https://nlp.johnsnowlabs.com/2023/03/30/summarizer_clinical_jsl_augmented_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_augmented_pipeline_en_4.4.2_3.0_1685392918075.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_augmented_pipeline_en_4.4.2_3.0_1685392918075.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_jsl_augmented_pipeline", "en", "clinical/models") text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_jsl_augmented_pipeline", "en", "clinical/models") val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ val result = pipeline.fullAnnotate(text) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_ft_conll2003","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_ft_conll2003","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_base_uncased_ft_conll2003| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sarahmiller137/distilbert-base-uncased-ft-conll2003 - https://aclanthology.org/W03-0419 - https://paperswithcode.com/sota?task=Token+Classification&dataset=conll2003 --- layout: model title: Bangla DistilBERT Embeddings (from neuralspace-reverie) author: John Snow Labs name: distilbert_embeddings_indic_transformers_bn_distilbert date: 2022-04-12 tags: [distilbert, embeddings, bn, open_source] task: Embeddings language: bn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `indic-transformers-bn-distilbert` is a Bangla model orginally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_indic_transformers_bn_distilbert_bn_3.4.2_3.0_1649783483143.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_indic_transformers_bn_distilbert_bn_3.4.2_3.0_1649783483143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_indic_transformers_bn_distilbert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|248.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-bn-distilbert - https://oscar-corpus.com/ --- layout: model title: Legal Death Clause Binary Classifier author: John Snow Labs name: legclf_death_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `death` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `death` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_death_clause_en_1.0.0_3.2_1660122314869.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_death_clause_en_1.0.0_3.2_1660122314869.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_death_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[death]| |[other]| |[other]| |[death]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_death_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support death 0.98 0.95 0.96 43 other 0.98 0.99 0.99 112 accuracy - - 0.98 155 macro-avg 0.98 0.97 0.98 155 weighted-avg 0.98 0.98 0.98 155 ``` --- layout: model title: Translate English to Atlantic-Congo languages Pipeline author: John Snow Labs name: translate_en_alv date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, alv, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `alv` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_alv_xx_2.7.0_2.4_1609691115946.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_alv_xx_2.7.0_2.4_1609691115946.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_alv", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_alv", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.alv').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_alv| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Recognize Entities OntoNotes - BERT Mini author: John Snow Labs name: onto_recognize_entities_bert_mini date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [en, pipeline, open_source] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `small_bert_L4_256` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_mini_en_2.7.0_2.4_1607510918214.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_bert_mini_en_2.7.0_2.4_1607510918214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_bert_base') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_bert_base") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu text = ["""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament."""] ner_df = nlu.load('en.ner.onto.bert.mini').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |Parliament |ORG | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_bert_mini| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: Legal Leather And Textile Industries Document Classifier (EURLEX) author: John Snow Labs name: legclf_leather_and_textile_industries_bert date: 2023-03-06 tags: [en, legal, classification, clauses, leather_and_textile_industries, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_leather_and_textile_industries_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Leather_and_Textile_Industries or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Leather_and_Textile_Industries`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_leather_and_textile_industries_bert_en_1.0.0_3.0_1678111851690.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_leather_and_textile_industries_bert_en_1.0.0_3.0_1678111851690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_leather_and_textile_industries_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Leather_and_Textile_Industries]| |[Other]| |[Other]| |[Leather_and_Textile_Industries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_leather_and_textile_industries_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Leather_and_Textile_Industries 0.88 0.95 0.91 135 Other 0.95 0.90 0.92 162 accuracy - - 0.92 297 macro-avg 0.92 0.92 0.92 297 weighted-avg 0.92 0.92 0.92 297 ``` --- layout: model title: Fast Neural Machine Translation Model from Welsh to English author: John Snow Labs name: opus_mt_cy_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, cy, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `cy` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_cy_en_xx_2.7.0_2.4_1609164073979.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_cy_en_xx_2.7.0_2.4_1609164073979.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_cy_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Zulu RobertaForMaskedLM Cased model (from MoseliMotsoehli) author: John Snow Labs name: roberta_embeddings_zuberta date: 2022-12-12 tags: [zu, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: zu edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `zuBERTa` is a Zulu model originally trained by `MoseliMotsoehli`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_zuberta_zu_4.2.4_3.0_1670859948117.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_zuberta_zu_4.2.4_3.0_1670859948117.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_zuberta","zu") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_zuberta","zu") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_zuberta| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zu| |Size:|256.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/MoseliMotsoehli/zuBERTa - https://wortschatz.uni-leipzig.de/en/download - https://zu.wikipedia.org/wiki/Special:AllPages --- layout: model title: Detect Disease Mentions in Spanish (Tweets) author: John Snow Labs name: disease_mentions_tweet date: 2022-08-14 tags: [es, clinical, licensed, public_health, ner, disease, tweet] task: Named Entity Recognition language: es edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Named Entity Recognition model is intended for detecting disease mentions in Spanish tweets and trained by using MedicalNerApproach annotator that allows to train generic NER models based on Neural Networks. The model detects ENFERMEDAD. ## Predicted Entities `ENFERMEDAD` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_NER_DISEASE_ES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4TC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/disease_mentions_tweet_es_4.0.2_3.0_1660443994563.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/disease_mentions_tweet_es_4.0.2_3.0_1660443994563.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('disease_mentions_tweet', "es", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""El diagnóstico fueron varios. Principal: Neumonía en el pulmón derecho. Sinusitis de caballo, Faringitis aguda e infección de orina, también elevada. Gripe No. Estuvo hablando conmigo, sin exagerar, mas de media hora, dándome ánimo y fuerza y que sabe, porque ha visto"""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("disease_mentions_tweet", "es", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""El diagnóstico fueron varios. Principal: Neumonía en el pulmón derecho. Sinusitis de caballo, Faringitis aguda e infección de orina, también elevada. Gripe No. Estuvo hablando conmigo, sin exagerar, mas de media hora, dándome ánimo y fuerza y que sabe, porque ha visto""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.tweet_disease_mention").predict("""El diagnóstico fueron varios. Principal: Neumonía en el pulmón derecho. Sinusitis de caballo, Faringitis aguda e infección de orina, también elevada. Gripe No. Estuvo hablando conmigo, sin exagerar, mas de media hora, dándome ánimo y fuerza y que sabe, porque ha visto""") ```

## Results ```bash +---------------------+----------+ |chunk |ner_label | +---------------------+----------+ |Neumonía en el pulmón|ENFERMEDAD| |Sinusitis de caballo |ENFERMEDAD| |Faringitis aguda |ENFERMEDAD| |infección de orina |ENFERMEDAD| |Gripe |ENFERMEDAD| +---------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|disease_mentions_tweet| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|15.2 MB| ## References The dataset is Covid-19-specific and consists of tweets collected via a series of keywords associated with that disease. ## Benchmarking ```bash label precision recall f1-score support B-ENFERMEDAD 0.94 0.96 0.95 4243 I-ENFERMEDAD 0.83 0.77 0.80 1570 micro-avg 0.91 0.91 0.91 5813 macro-avg 0.88 0.87 0.87 5813 weighted-avg 0.91 0.91 0.91 5813 ``` --- layout: model title: Word2Vec Embeddings in Slovak (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, sk, open_source] task: Embeddings language: sk edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sk_3.4.1_3.0_1647457801748.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sk_3.4.1_3.0_1647457801748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("albert_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("albert_base_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.")) ```

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PER | |John Snow Labs|ORG | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_base_token_classifier_conll03_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|43.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - AlbertForTokenClassification - NerConverter - Finisher --- layout: model title: Legal Expenses Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_expenses_bert date: 2023-03-05 tags: [en, legal, classification, clauses, expenses, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Expenses` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Expenses`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_expenses_bert_en_1.0.0_3.0_1678050658714.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_expenses_bert_en_1.0.0_3.0_1678050658714.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_expenses_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Expenses]| |[Other]| |[Other]| |[Expenses]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_expenses_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Expenses 0.93 0.97 0.95 143 Other 0.98 0.94 0.96 173 accuracy - - 0.95 316 macro-avg 0.95 0.95 0.95 316 weighted-avg 0.95 0.95 0.95 316 ``` --- layout: model title: English image_classifier_vit_pasta_shapes ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_pasta_shapes date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pasta_shapes` is a English model originally trained by nateraw. ## Predicted Entities `corgi`, `husky`, `shibu inu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pasta_shapes_en_4.1.0_3.0_1660170092691.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pasta_shapes_en_4.1.0_3.0_1660170092691.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pasta_shapes", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pasta_shapes", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pasta_shapes| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForTokenClassification Cased model (from yanekyuk) author: John Snow Labs name: bert_token_classifier_cased_keyword_discriminator date: 2022-11-30 tags: [en, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-cased-keyword-discriminator` is a English model originally trained by `yanekyuk`. ## Predicted Entities `CON`, `ENT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_cased_keyword_discriminator_en_4.2.4_3.0_1669815162301.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_cased_keyword_discriminator_en_4.2.4_3.0_1669815162301.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_cased_keyword_discriminator","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_cased_keyword_discriminator","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_cased_keyword_discriminator| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/yanekyuk/bert-cased-keyword-discriminator --- layout: model title: Embeddings Clinical (Large) author: John Snow Labs name: embeddings_clinical_large date: 2023-04-07 tags: [licensed, en, clinical, embeddings] task: Embeddings language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on a list of clinical and biomedical datasets curated in-house, using the word2vec algorithm. The dataset curation cut-off date is March 2023 and the model is expected to have a better generalization on recent content. The size of the model is around 2 GB and has 200 dimensions. Our benchmark tests indicate that our legacy clinical embeddings (embeddings_clinical) can be replaced with this one while training a new model (existing/previous models will still need to use the legacy embeddings that they're trained with). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_large_en_4.3.2_3.0_1680905541704.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_large_en_4.3.2_3.0_1680905541704.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large","en","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_large","en","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("word_embeddings") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|embeddings_clinical_large| |Type:|embeddings| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|en| |Size:|2.0 GB| |Case sensitive:|true| |Dimension:|200| --- layout: model title: Translate English to Irish Pipeline author: John Snow Labs name: translate_en_ga date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ga, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ga` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ga_xx_2.7.0_2.4_1609700791565.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ga_xx_2.7.0_2.4_1609700791565.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ga", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ga", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ga').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ga| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Indemnity Clause Binary Classifier author: John Snow Labs name: legclf_indemnity_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `indemnity` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `indemnity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_clause_en_1.0.0_3.2_1660122519881.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indemnity_clause_en_1.0.0_3.2_1660122519881.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_indemnity_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[indemnity]| |[other]| |[other]| |[indemnity]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnity_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash legal precision recall f1-score support indemnity 0.96 0.92 0.94 25 other 0.97 0.98 0.98 64 accuracy - - 0.97 89 macro-avg 0.96 0.95 0.96 89 weighted-avg 0.97 0.97 0.97 89 ``` --- layout: model title: Pipeline to Detect biological concepts (biobert) author: John Snow Labs name: ner_bionlp_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_bionlp_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_bionlp_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_pipeline_en_3.4.1_3.0_1647871746678.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_pipeline_en_3.4.1_3.0_1647871746678.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_bionlp_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.") ``` ```scala val pipeline = new PretrainedPipeline("ner_bionlp_biobert_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.bionlp_biobert.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""") ```

## Results ```bash +-----------------------------------+--------------------+ |chunk |ner_label | +-----------------------------------+--------------------+ |human |Organism | |KCNJ9 |Gene_or_gene_product| |Kir 3.3 |Gene_or_gene_product| |GIRK3 |Gene_or_gene_product| |rectifying potassium (GIRK) channel|Gene_or_gene_product| |KCNJ9 |Gene_or_gene_product| |chromosome 1q21-23 |Cellular_component | |humantissues |Organism | |pancreas |Organ | |tissues |Tissue | |fat andskeletal muscle |Tissue | |KCNJ9 |Gene_or_gene_product| |KCNJ9 |Gene_or_gene_product| +-----------------------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bionlp_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: BERT Embeddings trained on MEDLINE/PubMed author: John Snow Labs name: bert_pubmed date: 2021-08-30 tags: [en, bert_embeddings, medline_pubmed_dataset, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture[1] pretrained from scratch on MEDLINE/PubMed This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings that improve its accuracy over the original BERT base checkpoint {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pubmed_en_3.2.0_3.0_1630316760578.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pubmed_en_3.2.0_3.0_1630316760578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_pubmed", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_pubmed", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.pubmed').predict(text, output_level='token') embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pubmed| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [MEDLINE/PubMed dataset](https://www.nlm.nih.gov/databases/download/pubmed_medline.html) This Model has been imported from: https://tfhub.dev/google/experts/bert/pubmed/2 --- layout: model title: Word Embeddings for Bengali (bengali_cc_300d) author: John Snow Labs name: bengali_cc_300d date: 2021-02-10 task: Embeddings language: bn edition: Spark NLP 2.7.3 spark_version: 2.4 tags: [open_source, bn, embeddings] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Common Crawl and Wikipedia using fastText. It is trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives. The model gives 300 dimensional vector outputs per token. The output vectors map words into a meaningful space where the distance between the vectors is related to semantic similarity of words. These embeddings can be used in multiple tasks like semantic word similarity, named entity recognition, sentiment analysis, and classification. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bengali_cc_300d_bn_2.7.3_2.4_1612956925175.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bengali_cc_300d_bn_2.7.3_2.4_1612956925175.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = WordEmbeddingsModel.pretrained("bengali_cc_300d", "bn") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("bn.embed").predict("""Put your text here.""") ```

## Results ```bash The model gives 300 dimensional feature vector output per token. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bengali_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 2.7.3+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[word_embeddings]| |Language:|bn| |Case sensitive:|false| |Dimension:|300| ## Data Source This model is imported from https://fasttext.cc/docs/en/crawl-vectors.html --- layout: model title: English DistilBertForQuestionAnswering Cased model (from rowan1224) author: John Snow Labs name: distilbert_qa_slp date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-slp` is a English model originally trained by `rowan1224`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_slp_en_4.3.0_3.0_1672774282530.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_slp_en_4.3.0_3.0_1672774282530.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_slp","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_slp","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_slp| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/rowan1224/distilbert-slp --- layout: model title: Fast and Accurate Language Identification - 21 Languages (CNN) author: John Snow Labs name: ld_wiki_tatoeba_cnn_21 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, language_detection, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The model is trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Bulgarian`, `Czech`, `Danish`, `German`, `Greek`, `English`, `Estonian`, `Finnish`, `French`, `Hungarian`, `Italian`, `Lithuanian`, `Latvian`, `Dutch`, `Polish`, `Portuguese`, `Romanian`, `Slovak`, `Slovenian`, `Spanish`, `Swedish`. ## Predicted Entities `bg`, `cs`, `da`, `de`, `el`, `en`, `et`, `fi`, `fr`, `hu`, `it`, `lt`, `lv`, `nl`, `pl`, `pt`, `ro`, `sk`, `sl`, `es`, `sv`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_21_xx_2.7.0_2.4_1607177877570.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_21_xx_2.7.0_2.4_1607177877570.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_21", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_21", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_21').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_tatoeba_cnn_21| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia and Tatoeba. ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | de| 1000| 1000| 1.0| | nl| 1000| 1000| 1.0| | pt| 1000| 1000| 1.0| | fr| 1000| 1000| 1.0| | es| 1000| 1000| 1.0| | it| 1000| 1000| 1.0| | fi| 1000| 1000| 1.0| | da| 1000| 999| 0.999| | en| 1000| 999| 0.999| | sv| 1000| 998| 0.998| | el| 1000| 996| 0.996| | bg| 1000| 989| 0.989| | pl| 914| 903|0.9879649890590809| | hu| 880| 867|0.9852272727272727| | ro| 784| 771|0.9834183673469388| | lt| 1000| 982| 0.982| | sk| 1000| 976| 0.976| | et| 928| 903|0.9730603448275862| | cs| 1000| 967| 0.967| | sl| 914| 875|0.9573304157549234| | lv| 916| 869|0.9486899563318777| +--------+-----+-------+------------------+ +-------+--------------------+ |summary| precision| +-------+--------------------+ | count| 21| | mean| 0.9876995879070323| | stddev|0.015446490915012105| | min| 0.9486899563318777| | max| 1.0| +-------+--------------------+ ``` --- layout: model title: German BertForSequenceClassification Base Cased model (from mrm8488) author: John Snow Labs name: bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx date: 2022-07-13 tags: [de, open_source, bert, sequence_classification] task: Text Classification language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-german-dbmdz-cased-finetuned-pawsx-de` is a German model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx_de_4.0.0_3.0_1657720863133.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx_de_4.0.0_3.0_1657720863133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = BertForSequenceClassification.pretrained("bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = BertForSequenceClassification.pretrained("bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx","de") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_base_german_dbmdz_cased_finetuned_pawsx| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|de| |Size:|412.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/bert-base-german-dbmdz-cased-finetuned-pawsx-de --- layout: model title: English XLMRobertaForTokenClassification Base Cased model (from asahi417) author: John Snow Labs name: xlmroberta_ner_tner_base_all_english date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tner-xlm-roberta-base-all-english` is a English model originally trained by `asahi417`. ## Predicted Entities `time`, `corporation`, `ordinal number`, `cardinal number`, `rna`, `geopolitical area`, `protein`, `product`, `percent`, `dna`, `disease`, `cell line`, `law`, `other`, `date`, `chemical`, `event`, `work of art`, `cell type`, `location`, `language`, `quantity`, `facility`, `organization`, `group`, `money`, `person` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_base_all_english_en_4.1.0_3.0_1660423323063.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_base_all_english_en_4.1.0_3.0_1660423323063.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_base_all_english","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_base_all_english","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tner_base_all_english| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|815.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/asahi417/tner-xlm-roberta-base-all-english - https://github.com/asahi417/tner --- layout: model title: English asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18 TFWav2Vec2ForCTC from jhonparra18 author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18` is a English model originally trained by jhonparra18. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18_en_4.2.0_3.0_1664019836794.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18_en_4.2.0_3.0_1664019836794.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_guarani_small_by_jhonparra18| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_company_all_903429548 date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-company_all-903429548` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `Company`, `OOV` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_all_903429548_en_4.3.0_3.0_1677881042384.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_company_all_903429548_en_4.3.0_3.0_1677881042384.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_all_903429548","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_company_all_903429548","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_company_all_903429548| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-company_all-903429548 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken TFWav2Vec2ForCTC from cuzeverynameistaken author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken` is a English model originally trained by cuzeverynameistaken. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken_en_4.2.0_3.0_1664023081884.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken_en_4.2.0_3.0_1664023081884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab1_by_cuzeverynameistaken| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English BertForQuestionAnswering model (from andi611) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-squad2-with-ner-Pistherea-conll2003-with-neg-with-repeat` is a English model orginally trained by `andi611`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat_en_4.0.0_3.0_1654537307579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat_en_4.0.0_3.0_1654537307579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_conll.bert.large_uncased_pistherea.by_andi611").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_Pistherea_conll2003_with_neg_with_repeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/andi611/bert-large-uncased-whole-word-masking-squad2-with-ner-Pistherea-conll2003-with-neg-with-repeat --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes (sbiobertresolve_icd10cm_slim_billable_hcc) author: John Snow Labs name: sbiobertresolve_icd10cm_slim_billable_hcc date: 2021-08-26 tags: [en, licensed] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.3 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to ICD10 CM codes using sentence biobert embeddings. In this model, synonyms having low cosine similarity to unnormalized terms are dropped. It also returns the official resolution text within the brackets inside the metadata. The model is augmented with synonyms, and previous augmentations are flexed according to cosine distances to unnormalized terms (ground truths). ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for aux_label parameter in the metadata to get HCC status. The HCC status can be divided to get further information: billable status, hcc status, and hcc score.For example, in the example shared below the billable status is 1, hcc status is 1, and hcc score is 11. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.1.3_2.4_1629971987395.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_3.1.3_2.4_1629971987395.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:---------------|:--------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|-------------------------------------------:|:----------------------------|:---------------------------------------------------------| | 0 | bladder cancer | C671 |[bladder cancer, dome [Malignant neoplasm of dome of bladder], cancer of the urinary bladder [Malignant neoplasm of bladder, unspecified], adenocarcinoma, bladder neck [Malignant neoplasm of bladder neck], cancer in situ of urinary bladder [Carcinoma in situ of bladder], cancer of the urinary bladder, ureteric orifice [Malignant neoplasm of ureteric orifice], tumor of bladder neck [Neoplasm of unspecified behavior of bladder], cancer of the urethra [Malignant neoplasm of urethra]]| [C671, C679, C675, D090, C676, D494, C680] | ['1', '1', '11'] | [0.0685, 0.0709, 0.0963, 0.0978, 0.1068, 0.1080, 0.1211] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: ICD10CM Puerile Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_puerile_clinical class: ChunkEntityResolverModel language: en nav_key: models repository: clinical/models date: 2020-04-28 task: Entity Resolution edition: Healthcare NLP 2.4.5 spark_version: 2.4 tags: [clinical,licensed,entity_resolution,en] deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_puerile_clinical_en_2.4.5_2.4_1588103916781.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_puerile_clinical_en_2.4.5_2.4_1588103916781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python ... puerile_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_puerile_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("resolution") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, puerile_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val puerile_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_puerile_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, puerile_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.h2_title} ## Results ```bash chunk entity icd10Puerile_description icd10Puerile_code 0 cold Symptom_Name Prolonged pregnancy O481 1 cough Symptom_Name Diseases of the respiratory system complicatin... O9953 2 runny nose Symptom_Name Other specified pregnancy related conditions, ... O26899 3 Mom Gender Classical hydatidiform mole O010 4 she Gender Other vomiting complicating pregnancy O218 5 no Negated Continuing pregnancy after spontaneous abortio... O3111X0 6 fever Symptom_Name Puerperal sepsis O85 7 Her Gender Obesity complicating the puerperium O99215 8 she Gender Other vomiting complicating pregnancy O218 9 spitting up a lot Symptom_Name Eclampsia, unspecified as to time period O159 10 She Gender Other vomiting complicating pregnancy O218 11 no Negated Continuing pregnancy after spontaneous abortio... O3111X0 12 difficulty breathing Symptom_Name Other disorders of lactation O9279 13 her Gender Diseases of the nervous system complicating th... O99355 14 cough Symptom_Name Diseases of the respiratory system complicatin... O9953 15 dry Modifier Cracked nipple associated with lactation O9213 16 hacky Modifier Severe pre-eclampsia, unspecified trimester O1410 17 She Gender Other vomiting complicating pregnancy O218 18 fairly Modifier Maternal care for high head at term, not appli... O324XX0 19 congested Symptom_Name Postpartum thyroiditis O905 20 She Gender Other vomiting complicating pregnancy O218 21 Amoxil Drug_Name Suppressed lactation O925 22 Aldex Drug_Name Severe pre-eclampsia, unspecified trimester O1410 23 her Gender Diseases of the nervous system complicating th... O99355 24 Mom Gender Classical hydatidiform mole O010 25 she Gender Other vomiting complicating pregnancy O218 26 She Gender Other vomiting complicating pregnancy O218 27 difficulty breathing Symptom_Name Other disorders of lactation O9279 28 She Gender Other vomiting complicating pregnancy O218 29 congested and her appetite had decreased Symptom_Name Decreased fetal movements, second trimester, n... O368120 30 She Gender Other vomiting complicating pregnancy O218 31 102.6 Temperature Postpartum coagulation defects O723 32 trouble sleeping Symptom_Name Other disorders of lactation O9279 33 secondary to Modifier Unspecified pre-existing hypertension complica... O10919 34 congestion Symptom_Name Viral hepatitis complicating childbirth O9842 ``` {:.model-param} ## Model Information {:.table-model} |----------------|---------------------------------------| | Name: | chunkresolve_icd10cm_puerile_clinical | | Type: | ChunkEntityResolverModel | | Compatibility: | Spark NLP 2.4.5+ | | License: | Licensed | |Edition:|Official| | |Input labels: | [token, chunk_embeddings] | |Output labels: | [entity] | | Language: | en | | Case sensitive: | True | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on ICD10CM Dataset Range: O0000-O9989 https://www.icd10data.com/ICD10CM/Codes/O00-O9A --- layout: model title: Finnish BertForMaskedLM Base Uncased model (from TurkuNLP) author: John Snow Labs name: bert_embeddings_base_finnish_uncased_v1 date: 2022-12-02 tags: [fi, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: fi edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-finnish-uncased-v1` is a Finnish model originally trained by `TurkuNLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_finnish_uncased_v1_fi_4.2.4_3.0_1670017573662.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_finnish_uncased_v1_fi_4.2.4_3.0_1670017573662.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_finnish_uncased_v1","fi") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_finnish_uncased_v1","fi") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_finnish_uncased_v1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fi| |Size:|467.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1 - http://dl.turkunlp.org/finbert/bert-base-finnish-cased-v1.zip - http://dl.turkunlp.org/finbert/bert-base-finnish-uncased-v1.zip - https://arxiv.org/abs/1912.07076 - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/master/multilingual.md - https://raw.githubusercontent.com/TurkuNLP/FinBERT/master/img/yle-ylilauta-curves.png - https://fasttext.cc/ - https://github.com/spyysalo/finbert-text-classification - https://github.com/spyysalo/yle-corpus - https://github.com/spyysalo/ylilauta-corpus - https://arxiv.org/abs/1908.04212 - https://github.com/Traubert/FiNer-rules - https://arxiv.org/pdf/1908.04212.pdf - https://github.com/jouniluoma/keras-bert-ner - https://github.com/mpsilfve/finer-data - https://universaldependencies.org/ - https://github.com/spyysalo/bert-pos - http://hdl.handle.net/11234/1-2837 - http://dl.turkunlp.org/finbert/bert-base-finnish-uncased.zip - http://dl.turkunlp.org/finbert/bert-base-finnish-cased.zip --- layout: model title: Legal Rights Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_rights_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, rights, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_rights_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `rights-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `rights-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_rights_agreement_bert_en_1.0.0_3.0_1669369164449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_rights_agreement_bert_en_1.0.0_3.0_1669369164449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_rights_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[rights-agreement]| |[other]| |[other]| |[rights-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_rights_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.91 0.97 0.94 65 rights-agreement 0.92 0.80 0.86 30 accuracy - - 0.92 95 macro-avg 0.92 0.88 0.90 95 weighted-avg 0.92 0.92 0.91 95 ``` --- layout: model title: Part of Speech for Armenian author: John Snow Labs name: pos_ud_armtdp date: 2021-03-09 tags: [part_of_speech, open_source, armenian, pos_ud_armtdp, hy] task: Part of Speech Tagging language: hy edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - PROPN - NOUN - NUM - PUNCT - ADJ - VERB - ADP - ADV - CCONJ - X - AUX - DET - PRON - SCONJ - SYM - PART - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_armtdp_hy_3.0.0_3.0_1615292139311.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_armtdp_hy_3.0.0_3.0_1615292139311.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_armtdp", "hy") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Ողջույն, John ոն Ձյուն լաբորատորիաներից: ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_armtdp", "hy") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Ողջույն, John ոն Ձյուն լաբորատորիաներից: ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Ողջույն, John ոն Ձյուն լաբորատորիաներից: ""] token_df = nlu.load('hy.pos').predict(text) token_df ```

## Results ```bash token pos 0 Ողջույն INTJ 1 , PUNCT 2 John ADJ 3 ոն NOUN 4 Ձյուն NOUN 5 լաբորատորիաներից NOUN 6 : PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_armtdp| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|hy| --- layout: model title: Medical Text Generation (T5-based) author: John Snow Labs name: text_generator_generic_flan_base date: 2023-04-03 tags: [licensed, en, clinical, text_generation, tensorflow] task: Text Generation language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalTextGenerator article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based text generation model, which is basically the same as official [Flan-T5-base model](https://huggingface.co/google/flan-t5-base) released by Google. Given a few tokens as an intro, it can generate human-like, conceptually meaningful texts up to 512 tokens given an input text (max 1024 tokens). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/MEDICAL_TEXT_GENERATION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/33.1.Medical_Text_Generation.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/text_generator_generic_flan_base_en_4.3.2_3.0_1680522827259.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/text_generator_generic_flan_base_en_4.3.2_3.0_1680522827259.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("prompt")\ .setOutputCol("document_prompt") med_text_generator = MedicalTextGenerator.pretrained("text_generator_generic_flan_base", "en", "clinical/models")\ .setInputCols("document_prompt")\ .setOutputCol("answer")\ .setMaxNewTokens(256)\ .setDoSample(True)\ .setTopK(3)\ .setRandomSeed(42) pipeline = Pipeline(stages=[document_assembler, med_text_generator]) data = spark.createDataFrame([["the patient is admitted to the clinic with a severe back pain and "]]).toDF("prompt") pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("prompt") .setOutputCol("document_prompt") val med_text_generator = MedicalTextGenerator.pretrained("text_generator_generic_flan_base", "en", "clinical/models") .setInputCols("document_prompt") .setOutputCol("answer") .setMaxNewTokens(256) .setDoSample(true) .setTopK(3) .setRandomSeed(42) val pipeline = new Pipeline().setStages(Array(document_assembler, med_text_generator)) val data = Seq(Array("the patient is admitted to the clinic with a severe back pain and ")).toDS.toDF("prompt") val result = pipeline.fit(data).transform(data) ```

## Results ```bash ['the patient is admitted to the clinic with a severe back pain and a severe left - sided leg pain. The patient was diagnosed with a lumbar disc herniation and underwent a discectomy. The patient was discharged on the third postoperative day. The patient was followed up for a period of 6 months and was found to be asymptomatic. A rare case of a giant cell tumor of the sacrum. Giant cell tumors ( GCTs ) are benign, locally aggressive tumors that are most commonly found in the long bones of the extremities. They are rarely found in the spine. We report a case of a GCT of the sacrum in a young female patient. The patient presented with a history of progressive lower back pain and a palpable mass in the left buttock. The patient underwent a left hemilaminectomy and biopsy. The histopathological examination revealed a GCT. The patient was treated with a combination of surgery and radiation therapy. The patient was followed up for 2 years and no recurrence was observed. A rare case of a giant cell tumor of the sacrum. Giant cell tumors ( GCTs ) are benign, locally aggressive tumors that are most commonly found in the long bones of the extremities. They are rarely found in the spine. We report a case of a GCT'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|text_generator_generic_flan_base| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.9 MB| --- layout: model title: Extract relations between phenotypic abnormalities and diseases (ReDL) author: John Snow Labs name: redl_human_phenotype_gene_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations to fully understand the origin of some phenotypic abnormalities and their associated diseases. `1` : Entities are related, `0` : Entities are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_2.7.3_2.4_1612440673031.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_2.7.3_2.4_1612440673031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|-----------:|:----------|----------------:|--------------:|:---------------------|:----------|----------------:|--------------:|:--------------------|-------------:| | 0 | 0 | HP | 10 | 29 | retinal degeneration | HP | 32 | 43 | hearing loss | 0.893809 | | 1 | 0 | HP | 10 | 29 | retinal degeneration | HP | 49 | 61 | renal failure | 0.958486 | | 2 | 1 | HP | 10 | 29 | retinal degeneration | HP | 162 | 180 | autosomal recessive | 0.65584 | | 3 | 0 | HP | 32 | 43 | hearing loss | HP | 64 | 76 | short stature | 0.707055 | | 4 | 1 | HP | 32 | 43 | hearing loss | GENE | 96 | 103 | SH3PXD2B | 0.640802 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_human_phenotype_gene_biobert| |Compatibility:| Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Data Source Trained on a silver standard corpus of human phenotype and gene annotations and their relations. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.922 0.908 0.915 129 1 0.831 0.855 0.843 71 Avg. 0.877 0.882 0.879 ``` --- layout: model title: English DistilBertForQuestionAnswering model (from Raphaelg9) author: John Snow Labs name: distilbert_qa_Raphaelg9_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Raphaelg9`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Raphaelg9_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724387431.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Raphaelg9_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724387431.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Raphaelg9_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Raphaelg9_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Raphaelg9").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Raphaelg9_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Raphaelg9/distilbert-base-uncased-finetuned-squad --- layout: model title: English BertForQuestionAnswering model (from fractalego) author: John Snow Labs name: bert_qa_fewrel_zero_shot date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fewrel-zero-shot` is a English model orginally trained by `fractalego`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_fewrel_zero_shot_en_4.0.0_3.0_1654187638832.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_fewrel_zero_shot_en_4.0.0_3.0_1654187638832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_fewrel_zero_shot","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_fewrel_zero_shot","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.zero_shot").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_fewrel_zero_shot| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/fractalego/fewrel-zero-shot - https://www.aclweb.org/anthology/2020.coling-main.124 --- layout: model title: Embeddings Scielo 300 dims author: John Snow Labs name: embeddings_scielo_300d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-26 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_300d_es_2.5.0_2.4_1590467138742.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_scielo_300d_es_2.5.0_2.4_1590467138742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.scielo.300d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_scielo_300d``. {:.model-param} ## Model Information {:.table-model} |---------------|------------------------| | Name: | embeddings_scielo_300d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | es | | Dimension: | 300.0 | {:.h2_title} ## Data Source Trained on Scielo Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: Stopwords Remover for Hindi language (233 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, hi, open_source] task: Stop Words Removal language: hi edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_hi_3.4.1_3.0_1646672942184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_hi_3.4.1_3.0_1646672942184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+ |result | +-------------------+ |[तुम, मुझसे, बेहतर]| +-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|hi| |Size:|2.2 KB| --- layout: model title: English DistilBertForQuestionAnswering model (from jgammack) MTL author: John Snow Labs name: distilbert_qa_MTL_base_uncased_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `MTL-distilbert-base-uncased-squad` is a English model originally trained by `jgammack`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_MTL_base_uncased_squad_en_4.0.0_3.0_1654722966366.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_MTL_base_uncased_squad_en_4.0.0_3.0_1654722966366.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_MTL_base_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_MTL_base_uncased_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased_mtl.by_jgammack").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_MTL_base_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/jgammack/MTL-distilbert-base-uncased-squad --- layout: model title: Translate English to Lunda Pipeline author: John Snow Labs name: translate_en_lun date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, lun, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `lun` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_lun_xx_2.7.0_2.4_1609687838269.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_lun_xx_2.7.0_2.4_1609687838269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_lun", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_lun", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.lun').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_lun| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Arabic to English author: John Snow Labs name: opus_mt_ar_en date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, en, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: en {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_en_xx_3.1.0_2.4_1622552419647.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_en_xx_3.1.0_2.4_1622552419647.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_world_landmarks ViTForImageClassification from mmgyorke author: John Snow Labs name: image_classifier_vit_world_landmarks date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_world_landmarks` is a English model originally trained by mmgyorke. ## Predicted Entities `la sagrada familia`, `leaning tower of pisa`, `arc de triomphe`, `taj mahal`, `big ben` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_world_landmarks_en_4.1.0_3.0_1660171257315.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_world_landmarks_en_4.1.0_3.0_1660171257315.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_world_landmarks", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_world_landmarks", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_world_landmarks| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Option Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_option_agreement date: 2022-12-06 tags: [en, legal, classification, agreement, option, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_option_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `option-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `option-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_option_agreement_en_1.0.0_3.0_1670357991770.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_option_agreement_en_1.0.0_3.0_1670357991770.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[option-agreement]| |[other]| |[other]| |[option-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_option_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support option-agreement 0.96 0.93 0.95 59 other 0.96 0.98 0.97 111 accuracy - - 0.96 170 macro-avg 0.96 0.96 0.96 170 weighted-avg 0.96 0.96 0.96 170 ``` --- layout: model title: Detect Drugs - Generalized Single Entity (ner_drugs_greedy) author: John Snow Labs name: ner_drugs_greedy date: 2020-12-14 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 2.6.5 spark_version: 2.4 tags: [ner, licensed, en, clinical] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a single entity model that generalises all posology concepts into one and finds longest available chunks of drugs. It is trained using `embeddings_clinical` so please use the same embeddings in the pipeline. ## Predicted Entities `DRUG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugs_greedy_en_2.6.4_2.4_1607417409084.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugs_greedy_en_2.6.4_2.4_1607417409084.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_drugs_greedy", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated."]]).toDF("text")) ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = NerDLModel.pretrained("ner_drugs_greedy", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter)) val data = Seq("DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.drugsgreedy").predict("""DOSAGE AND ADMINISTRATION The initial dosage of hydrocortisone tablets may vary from 20 mg to 240 mg of hydrocortisone per day depending on the specific disease entity being treated.""") ```

## Results ```bash +-----------------------------------+------------+ | chunk | ner_label | +-----------------------------------+------------+ | hydrocortisone tablets | DRUG | | 20 mg to 240 mg of hydrocortisone | DRUG | +-----------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_greedy| |Type:|ner| |Compatibility:|Spark NLP 2.6.5+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Dependencies:|embeddings_clinical| ## Data Source Trained on augmented i2b2_med7 + FDA dataset with ``embeddings_clinical``, [https://www.i2b2.org/NLP/Medication](https://www.i2b2.org/NLP/Medication). ## Benchmarking ```bash label tp fp fn prec rec f1 I-DRUG 37858 4166 3338 0.90086615 0.91897273 0.9098294 B-DRUG 29926 2006 1756 0.937179 0.9445742 0.9408621 tp: 67784 fp: 6172 fn: 5094 labels: 2 Macro-average prec: 0.91902256, rec: 0.9317734, f1: 0.92535406 Micro-average prec: 0.916545, rec: 0.93010235, f1: 0.9232739 ``` --- layout: model title: English Bert Embeddings (from mlcorelib) author: John Snow Labs name: bert_embeddings_deberta_base_uncased date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `deberta-base-uncased` is a English model orginally trained by `mlcorelib`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_deberta_base_uncased_en_3.4.2_3.0_1649672445146.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_deberta_base_uncased_en_3.4.2_3.0_1649672445146.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_deberta_base_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|410.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/mlcorelib/deberta-base-uncased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: German asr_exp_w2v2t_vp_s184 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2t_vp_s184 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_vp_s184` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2t_vp_s184_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s184_de_4.2.0_3.0_1664109862866.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2t_vp_s184_de_4.2.0_3.0_1664109862866.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2t_vp_s184', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2t_vp_s184", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2t_vp_s184| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Explain Document Pipeline for German author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, german, explain_document_md, pipeline, de] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: de edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_de_3.0.0_3.0_1616431227526.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_de_3.0.0_3.0_1616431227526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'de') annotations = pipeline.fullAnnotate(""Hallo aus John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "de") val result = pipeline.fullAnnotate("Hallo aus John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hallo aus John Snow Labs! ""] result_df = nlu.load('de.explain.document').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-------------------------------|:------------------------------|:------------------------------------------|:------------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hallo aus John Snow Labs! '] | ['Hallo aus John Snow Labs!'] | ['Hallo', 'aus', 'John', 'Snow', 'Labs!'] | ['Hallo', 'aus', 'John', 'Snow', 'Labs!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.5910000205039978,.,...]] | ['O', 'O', 'I-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|de| --- layout: model title: Translate Danish to English Pipeline author: John Snow Labs name: translate_da_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, da, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `da` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_da_en_xx_2.7.0_2.4_1609689269793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_da_en_xx_2.7.0_2.4_1609689269793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_da_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_da_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.da.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_da_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-10` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10_en_4.0.0_3.0_1654191610651.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10_en_4.0.0_3.0_1654191610651.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_512d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_512_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|387.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-512-finetuned-squad-seed-10 --- layout: model title: Financial Relation Extraction (Work Experience, Sm, Bidirectional) author: John Snow Labs name: finre_work_experience date: 2022-09-28 tags: [work, experience, en, licensed] task: Relation Extraction language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole financial report. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `finclf_work_experience_item` Text Classifier to select only these paragraphs; This model allows you to analyzed present and past job positions of people, extracting relations between PERSON, ORG, ROLE and DATE. This model requires an NER with the mentioned entities, as `finner_org_per_role` and can also be combined with `finassertiondl_past_roles` to detect if the entities are mentioned to have happened in the PAST or not (although you can also infer that from the relations as `had_role_until`). This model is a `sm` model without meaningful directions in the relations (the model was not trained to understand if the direction of the relation is from left to right or right to left). There are bigger models in Models Hub trained also with directed relationships. ## Predicted Entities `has_role`, `had_role_until`, `has_role_from`, `works_for`, `has_role_in_company` {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finre_work_experience_en_1.0.0_3.0_1664360618647.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finre_work_experience_en_1.0.0_3.0_1664360618647.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","en")\ .setInputCols(["document"])\ .setOutputCol("sentence")\ tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained('finner_org_per_role_date', 'en', 'finance/models')\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") pos = nlp.PerceptronModel.pretrained()\ .setInputCols(["sentence", "token"])\ .setOutputCol("pos") dependency_parser = nlp.DependencyParserModel().pretrained("dependency_conllu", "en")\ .setInputCols(["sentence", "pos", "token"])\ .setOutputCol("dependencies") re_ner_chunk_filter = finance.RENerChunksFilter()\ .setInputCols(["ner_chunk", "dependencies"])\ .setOutputCol("re_ner_chunk")\ .setRelationPairs(["PERSON-ROLE, ORG-ROLE, DATE-ROLE, PERSON-ORG"])\ .setMaxSyntacticDistance(5) re_Model = finance.RelationExtractionDLModel.pretrained("finre_work_experience", "en", "finance/models")\ .setInputCols(["re_ner_chunk", "sentence"])\ .setOutputCol("relations")\ .setPredictionThreshold(0.5) pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter, pos, dependency_parser, re_ner_chunk_filter, re_Model ]) empty_df = spark.createDataFrame([['']]).toDF("text") re_model = pipeline.fit(empty_df) light_model = LightPipeline(re_model) text_list = ["""We have experienced significant changes in our senior management team over the past several years, including the appointments of Mark Schmitz as our Executive Vice President and Chief Operating Officer in 2019.""", """In January 2019, Jose Cil was assigned the CEO of Restaurant Brands International, and Daniel Schwartz was assigned the Executive Chairman of the company.""", ] results = light_model.fullAnnotate(text_list) ```

## Results ```bash has_role PERSON 129 140 Mark Schmitz ROLE 149 172 Executive Vice President 0.8707728 has_role PERSON 129 140 Mark Schmitz ROLE 178 200 Chief Operating Officer 0.97559035 has_role_from ROLE 149 172 Executive Vice President DATE 205 208 2019 0.9327241 has_role_from ROLE 178 200 Chief Operating Officer DATE 205 208 2019 0.90718126 has_role_from DATE 3 14 January 2019 ROLE 43 45 CEO 0.996639 has_role_from DATE 3 14 January 2019 ROLE 120 137 Executive Chairman 0.9964874 has_role PERSON 17 24 Jose Cil ROLE 43 45 CEO 0.8917691 has_role PERSON 17 24 Jose Cil ROLE 120 137 Executive Chairman 0.8527716 has_role ROLE 43 45 CEO PERSON 87 101 Daniel Schwartz 0.5765097 has_role PERSON 87 101 Daniel Schwartz ROLE 120 137 Executive Chairman 0.79235893 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finre_work_experience| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|409.9 MB| ## References Manual annotations on CUAD dataset, 10K filings and Wikidata ## Benchmarking ```bash label Recall Precision F1 Support had_role_until 0.972 0.972 0.972 36 has_role 0.986 0.980 0.983 146 has_role_from 0.983 0.983 0.983 58 has_role_in_company 0.954 0.969 0.961 65 works_for 0.933 0.933 0.933 15 Avg. 0.966 0.967 0.966 - Weighted-Avg. 0.975 0.975 0.975 - ``` --- layout: model title: Romanian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_romanian_legal date: 2023-02-16 tags: [ro, romanian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: ro edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-romanian-roberta-base` is a Romanian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_romanian_legal_ro_4.2.4_3.0_1676556391039.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_romanian_legal_ro_4.2.4_3.0_1676556391039.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_romanian_legal", "ro")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_romanian_legal", "ro") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_romanian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ro| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-romanian-roberta-base --- layout: model title: Legal Investment Subadvisory Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_investment_subadvisory_agreement_bert date: 2022-11-24 tags: [en, legal, classification, agreement, investment_subadvisory, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_investment_subadvisory_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `investment-subadvisory-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `investment-subadvisory-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_investment_subadvisory_agreement_bert_en_1.0.0_3.0_1669315177671.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_investment_subadvisory_agreement_bert_en_1.0.0_3.0_1669315177671.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_investment_subadvisory_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[investment-subadvisory-agreement]| |[other]| |[other]| |[investment-subadvisory-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_investment_subadvisory_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support investment-subadvisory-agreement 1.00 1.00 1.00 32 other 1.00 1.00 1.00 82 accuracy - - 1.00 114 macro-avg 1.00 1.00 1.00 114 weighted-avg 1.00 1.00 1.00 114 ``` --- layout: model title: English ElectraForQuestionAnswering model (from Palak) author: John Snow Labs name: electra_qa_google_base_discriminator_squad date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `google_electra-base-discriminator_squad` is a English model originally trained by `Palak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_google_base_discriminator_squad_en_4.0.0_3.0_1655921991596.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_google_base_discriminator_squad_en_4.0.0_3.0_1655921991596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_google_base_discriminator_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_google_base_discriminator_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.base.by_Palak").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_google_base_discriminator_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Palak/google_electra-base-discriminator_squad --- layout: model title: English XlmRoBertaForQuestionAnswering (from laifuchicago) author: John Snow Labs name: xlm_roberta_qa_farm2tran date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `farm2tran` is a English model originally trained by `laifuchicago`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_farm2tran_en_4.0.0_3.0_1655987409597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_farm2tran_en_4.0.0_3.0_1655987409597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_farm2tran","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_farm2tran","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.by_laifuchicago").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_farm2tran| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/laifuchicago/farm2tran --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from gkss) author: John Snow Labs name: distilbert_qa_gkss_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `gkss`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_gkss_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770818177.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_gkss_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770818177.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_gkss_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_gkss_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_gkss_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/gkss/distilbert-base-uncased-finetuned-squad --- layout: model title: Arabic T5ForConditionalGeneration Cased model (from malmarjeh) author: John Snow Labs name: t5_arabic_text_summarization date: 2023-01-30 tags: [ar, open_source, t5, tensorflow] task: Text Generation language: ar edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-arabic-text-summarization` is a Arabic model originally trained by `malmarjeh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_arabic_text_summarization_ar_4.3.0_3.0_1675107890347.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_arabic_text_summarization_ar_4.3.0_3.0_1675107890347.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_arabic_text_summarization","ar") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_arabic_text_summarization","ar") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_arabic_text_summarization| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ar| |Size:|1.4 GB| ## References - https://huggingface.co/malmarjeh/t5-arabic-text-summarization --- layout: model title: English DistilBertForQuestionAnswering model (from huxxx657) Jumbling author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_jumbling_squad_15 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-jumbling-squad-15` is a English model originally trained by `huxxx657`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_jumbling_squad_15_en_4.0.0_3.0_1654723967561.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_jumbling_squad_15_en_4.0.0_3.0_1654723967561.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_jumbling_squad_15","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_jumbling_squad_15","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased_v2.by_huxxx657").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_jumbling_squad_15| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/huxxx657/distilbert-base-uncased-finetuned-jumbling-squad-15 --- layout: model title: BERT Sentence Embeddings trained on MEDLINE/PubMed author: John Snow Labs name: sent_bert_pubmed date: 2021-08-31 tags: [en, open_source, sentence_embeddings, medline_pubmed_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture pretrained from scratch on MEDLINE/PubMed. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings that improve its accuracy over the original BERT base checkpoint. This model is intended to be used for a variety of English NLP tasks in the medical domain. The pre-training data contains more medical text and the model may not generalize to text outside of that domain. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_pubmed_en_3.2.0_3.0_1630412084893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_pubmed_en_3.2.0_3.0_1630412084893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_pubmed", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_pubmed", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.pubmed').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_pubmed| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [MEDLINE/PubMed dataset](https://www.nlm.nih.gov/databases/download/pubmed_medline.html) This Model has been imported from: https://tfhub.dev/google/experts/bert/pubmed/2 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from hcy11) author: John Snow Labs name: distilbert_qa_hcy11_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `hcy11`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_hcy11_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771148653.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_hcy11_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771148653.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hcy11_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hcy11_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hcy11_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hcy11/distilbert-base-uncased-finetuned-squad --- layout: model title: English DistilBertForTokenClassification Cased model (from TomUdale) author: John Snow Labs name: distilbert_token_classifier_sec_example date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sec_example` is a English model originally trained by `TomUdale`. ## Predicted Entities `PER`, `ORG`, `MISC`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_sec_example_en_4.3.1_3.0_1678783571147.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_sec_example_en_4.3.1_3.0_1678783571147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_sec_example","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_sec_example","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_sec_example| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/TomUdale/sec_example --- layout: model title: Multilingual BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp date: 2022-06-03 tags: [te, en, open_source, question_answering, bert, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-multi-cased-finedtuned-xquad-tydiqa-goldp` is a Multilingual model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp_xx_4.0.0_3.0_1654253530216.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp_xx_4.0.0_3.0_1654253530216.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.xquad_tydiqa.bert.cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_multi_cased_finedtuned_xquad_tydiqa_goldp| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-multi-cased-finedtuned-xquad-tydiqa-goldp --- layout: model title: Embeddings Sciwiki 300 dims author: John Snow Labs name: embeddings_sciwiki_300d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-27 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_sciwiki_300d_es_2.5.0_2.4_1590609454054.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_sciwiki_300d_es_2.5.0_2.4_1590609454054.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_sciwiki_300d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.sciwiki_300d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_sciwiki_300d``. {:.model-param} ## Model Information {:.table-model} |---------------|-------------------------| | Name: | embeddings_sciwiki_300d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | es | | Dimension: | 300.0 | {:.h2_title} ## Data Source Trained on Clinical Wikipedia Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: Legal Means Of Agricultural Production Document Classifier (EURLEX) author: John Snow Labs name: legclf_means_of_agricultural_production_bert date: 2023-03-06 tags: [en, legal, classification, clauses, means_of_agricultural_production, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_means_of_agricultural_production_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Means_of_Agricultural_Production or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Means_of_Agricultural_Production`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_means_of_agricultural_production_bert_en_1.0.0_3.0_1678111831055.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_means_of_agricultural_production_bert_en_1.0.0_3.0_1678111831055.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_means_of_agricultural_production_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Means_of_Agricultural_Production]| |[Other]| |[Other]| |[Means_of_Agricultural_Production]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_means_of_agricultural_production_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.9 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Means_of_Agricultural_Production 0.86 0.89 0.88 476 Other 0.87 0.83 0.85 416 accuracy - - 0.87 892 macro-avg 0.87 0.86 0.86 892 weighted-avg 0.87 0.87 0.87 892 ``` --- layout: model title: English asr_sp_proj TFWav2Vec2ForCTC from behroz author: John Snow Labs name: pipeline_asr_sp_proj date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_sp_proj` is a English model originally trained by behroz. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_sp_proj_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_sp_proj_en_4.2.0_3.0_1664019439184.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_sp_proj_en_4.2.0_3.0_1664019439184.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_sp_proj', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_sp_proj", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_sp_proj| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|349.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_small_ssm_nq date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-ssm-nq` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_ssm_nq_en_4.3.0_3.0_1675155920334.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_ssm_nq_en_4.3.0_3.0_1675155920334.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_ssm_nq","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_ssm_nq","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_ssm_nq| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|179.4 MB| ## References - https://huggingface.co/google/t5-small-ssm-nq - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/pdf/2002.08909.pdf - https://arxiv.org/abs/1910.10683.pdf - https://goo.gle/t5-cbqa - https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/how_much_know_ledge_image.png --- layout: model title: NER Pipeline for Hindi+English author: John Snow Labs name: bert_token_classifier_hi_en_ner_pipeline date: 2022-03-22 tags: [hindi, bert_token, hi, open_source] task: Named Entity Recognition language: hi edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on [bert_token_classifier_hi_en_ner](https://nlp.johnsnowlabs.com/2021/12/27/bert_token_classifier_hi_en_ner_hi.html). {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_HINDI_ENGLISH/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_HINDI_ENGLISH.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_hi_en_ner_pipeline_hi_3.4.1_3.0_1647954363761.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_hi_en_ner_pipeline_hi_3.4.1_3.0_1647954363761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_hi_en_ner_pipeline", lang = "hi") pipeline.annotate("रिलायंस इंडस्ट्रीज़ लिमिटेड (Reliance Industries Limited) एक भारतीय संगुटिका नियंत्रक कंपनी है, जिसका मुख्यालय मुंबई, महाराष्ट्र (Maharashtra) में स्थित है।रतन नवल टाटा (28 दिसंबर 1937, को मुम्बई (Mumbai), में जन्मे) टाटा समुह के वर्तमान अध्यक्ष, जो भारत की सबसे बड़ी व्यापारिक समूह है, जिसकी स्थापना जमशेदजी टाटा ने की और उनके परिवार की पीढियों ने इसका विस्तार किया और इसे दृढ़ बनाया।") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_hi_en_ner_pipeline", lang = "hi") val pipeline.annotate("रिलायंस इंडस्ट्रीज़ लिमिटेड (Reliance Industries Limited) एक भारतीय संगुटिका नियंत्रक कंपनी है, जिसका मुख्यालय मुंबई, महाराष्ट्र (Maharashtra) में स्थित है।रतन नवल टाटा (28 दिसंबर 1937, को मुम्बई (Mumbai), में जन्मे) टाटा समुह के वर्तमान अध्यक्ष, जो भारत की सबसे बड़ी व्यापारिक समूह है, जिसकी स्थापना जमशेदजी टाटा ने की और उनके परिवार की पीढियों ने इसका विस्तार किया और इसे दृढ़ बनाया।") ```

## Results ```bash +---------------------------+------------+ |chunk |ner_label | +---------------------------+------------+ |रिलायंस इंडस्ट्रीज़ लिमिटेड |ORGANISATION| |Reliance Industries Limited|ORGANISATION| |भारतीय |PLACE | |मुंबई |PLACE | |महाराष्ट्र |PLACE | |Maharashtra) |PLACE | |नवल टाटा |PERSON | |मुम्बई |PLACE | |Mumbai |PLACE | |टाटा समुह |ORGANISATION| |भारत |PLACE | |जमशेदजी टाटा |PERSON | +---------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_hi_en_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Language:|hi| |Size:|665.8 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - BertForTokenClassification - NerConverter - Finisher --- layout: model title: English DistilBertForTokenClassification Base Uncased model (from sarahmiller137) author: John Snow Labs name: distilbert_token_classifier_base_uncased_ft_conll2003 date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-ft-conll2003` is a English model originally trained by `sarahmiller137`. ## Predicted Entities `LOC`, `ORG`, `PER`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_uncased_ft_conll2003_en_4.3.0_3.0_1677881411947.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_base_uncased_ft_conll2003_en_4.3.0_3.0_1677881411947.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_ft_conll2003","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_base_uncased_ft_conll2003","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_base_uncased_ft_conll2003| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sarahmiller137/distilbert-base-uncased-ft-conll2003 - https://aclanthology.org/W03-0419 - https://paperswithcode.com/sota?task=Token+Classification&dataset=conll2003 --- layout: model title: English T5ForConditionalGeneration Cased model (from erickfm) author: John Snow Labs name: t5_neutrally date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `neutrally` is a English model originally trained by `erickfm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_neutrally_en_4.3.0_3.0_1675106407099.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_neutrally_en_4.3.0_3.0_1675106407099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_neutrally","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_neutrally","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_neutrally| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|923.9 MB| ## References - https://huggingface.co/erickfm/neutrally - https://github.com/rpryzant/neutralizing-bias - https://nlp.stanford.edu/pubs/pryzant2020bias.pdf - https://en.wikipedia.org/wiki/BLEU - https://apps-summer22.ischool.berkeley.edu/neutrally/ --- layout: model title: Spanish BERT Base Cased Embedding author: John Snow Labs name: bert_base_cased date: 2021-09-07 tags: [spanish, open_source, bert_embeddings, cased, es] task: Embeddings language: es edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description BETO is a BERT model trained on a big Spanish corpus. BETO is of size similar to a BERT-Base and was trained with the Whole Word Masking technique. Below you find Tensorflow and Pytorch checkpoints for the uncased and cased versions, as well as some results for Spanish benchmarks comparing BETO with Multilingual BERT as well as other (not BERT-based) models. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_base_cased_es_3.2.2_3.0_1630999631885.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_base_cased_es_3.2.2_3.0_1630999631885.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("bert_base_cased", "es") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = BertEmbeddings.pretrained("bert_base_cased", "es") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.bert.base_cased").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_keyphrase_extraction_distilbert_inspec| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/DeDeckerThomas/keyphrase-extraction-distilbert-inspec - https://paperswithcode.com/sota?task=Keyphrase+Extraction&dataset=inspec --- layout: model title: Legal License Agreement Document Binary Classifier (Longformer) author: John Snow Labs name: legclf_license_agreement date: 2022-12-18 tags: [en, legal, classification, licensed, document, longformer, license, agreement, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_license_agreement` model is a Longformer Document Classifier used to classify if the document belongs to the class `license-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If your document needs to process more than 4096 tokens, you can try the following: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. ## Predicted Entities `license-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_license_agreement_en_1.0.0_3.0_1671393662516.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_license_agreement_en_1.0.0_3.0_1671393662516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[license-agreement]| |[other]| |[other]| |[license-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_license_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support license-agreement 0.97 0.96 0.96 98 other 0.98 0.99 0.98 223 accuracy - - 0.98 321 macro-avg 0.98 0.97 0.97 321 weighted-avg 0.98 0.98 0.98 321 ``` --- layout: model title: English Named Entity Recognition (from kSaluja) author: John Snow Labs name: bert_ner_autonlp_tele_red_data_model_585716433 date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `autonlp-tele_red_data_model-585716433` is a English model orginally trained by `kSaluja`. ## Predicted Entities `TARGET`, `SUGGESTIONTYPE`, `CALLTYPE`, `INSTRUMENT`, `BUYPRICE`, `HOLDINGPERIOD`, `STOPLOSS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_autonlp_tele_red_data_model_585716433_en_3.4.2_3.0_1652096670651.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_autonlp_tele_red_data_model_585716433_en_3.4.2_3.0_1652096670651.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_autonlp_tele_red_data_model_585716433| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/kSaluja/autonlp-tele_red_data_model-585716433 --- layout: model title: Russian Fact Extraction NER author: John Snow Labs name: finner_bert_rufacts date: 2022-09-27 tags: [ru, licensed] task: Named Entity Recognition language: ru edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Russian Name Entity Recognition created for the RuREBus (Russian Relation Extraction for Business) shared task. The overall goal of the shared task was to develop business-oriented models capable of relation and/or fact extraction from texts. A lot of business applications require relation extraction. Although there are a few corpora, that contain texts annotated with relations, all of them are more of an academic nature and differ from typical business applications. There are a few reasons for this. First, the annotations are quite tight, i.e. almost every sentence contains an annotated relation. In contrast, the business-oriented documents often contain much less annotated examples. There might be one or two annotated examples in the whole document. Second, the existing corpora cover everyday topics (family relationships, birth and death relations, purchase and sale relations, etc). The business applications require other domain-specific relations. The goal of the task is to compare the methods for relation extraction in a more close-to-practice way. For these reasons, we suggest using the documents, produced by the Ministry of Economic Development of the Russian Federation. The corpus contains regional reports and strategic plans. A part of the corpus is annotated with named entities (8 classes) and semantic relations (11 classes). In total there are approximately 300 annotated documents. The annotation schema and the guidelines for annotators can be found in [here](https://github.com/dialogue-evaluation/RuREBus/blob/master/markup_instruction.pdf) (in Russian). The dataset consists of: 1. A train set with manually annotated named entities and relations. First and second parts of train set are avaliable [here](https://github.com/dialogue-evaluation/RuREBus/tree/master/train_data) 2. A large corpus (approx. 280 million tokens) of raw free-form documents, produced by the Ministry of Economic Development. These documents come from the same domain as the train and the test set. This data is avaliable [here](https://yadi.sk/d/9uKbo3p0ghdNpQ). 3. A test set without any annotations The predicted entities are: MET - Metric (productivity, growth...) ECO - Economical Entity / Concept (inner market, energy source...) BIN - 1-time action, binary (happened or not - construction, development, presence, absence...) CMP - Quantitative Comparision entity (increase, decrease...) QUA - Qualitative Comparison entity (stable, limited...) ACT - Activity (Restauration of buildings, Festivities in Cities...) INT - Institutions (Centers, Departments, etc) SOC - Social - Social object (Children, Elder people, Workers of X sector, ...) ## Predicted Entities `MET`, `ECO`, `BIN`, `CMP`, `QUA`, `ACT`, `INT`, `SOC` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FIN_NER_RUSSIAN_GOV/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_bert_rufacts_ru_1.0.0_3.0_1664277912886.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_bert_rufacts_ru_1.0.0_3.0_1664277912886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "ru") \ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") ner_model = finance.BertForTokenClassification.pretrained("finner_bert_rufacts", "en", "finance/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentencerDL, tokenizer, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) text_list = ["""В рамках обеспечения использования в деятельности ОМСУ муниципального образования Московской области региональных и муниципальных информационных систем предусматривается решение задач"""] import pandas as pd df = spark.createDataFrame(pd.DataFrame({"text" : text_list})) result = model.transform(df) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("ner_chunk"), F.expr("cols['1']['entity']").alias("label")).show(truncate = False) ```

## Results ```bash +--------------------------------------------------+-----+ |ner_chunk |label| +--------------------------------------------------+-----+ |обеспечения |BIN | |ОМСУ муниципального образования |INST | |региональных и муниципальных информационных систем|ECO | +--------------------------------------------------+-----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_bert_rufacts| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|ru| |Size:|358.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References https://github.com/dialogue-evaluation/RuREBus ## Benchmarking ```bash label precision recall f1-score support B-MET 0.7440 0.7440 0.7440 250 I-MET 0.8301 0.7704 0.7991 945 B-BIN 0.7248 0.7850 0.7537 614 B-ACT 0.6052 0.5551 0.5791 254 I-ACT 0.7215 0.6244 0.6695 892 B-ECO 0.6892 0.6813 0.6852 524 I-ECO 0.6750 0.6899 0.6824 861 B-CMP 0.8405 0.8354 0.8379 164 I-CMP 0.2000 0.0714 0.1053 14 B-INST 0.7152 0.7019 0.7085 161 I-INST 0.7560 0.7114 0.7330 440 B-SOC 0.5547 0.6698 0.6068 212 I-SOC 0.6178 0.7087 0.6601 381 B-QUA 0.6167 0.7303 0.6687 152 I-QUA 0.7333 0.4400 0.5500 25 micro-avg 0.7107 0.7017 0.7062 5927 macro-avg 0.6610 0.6337 0.6413 5927 weighted-avg 0.7136 0.7017 0.7059 5927 ``` --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from SauravMaheshkar) author: John Snow Labs name: distilbert_qa_finetuned_for_xqua_on_chaii date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-multi-finetuned-for-xqua-on-chaii` is a Multilingual model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_finetuned_for_xqua_on_chaii_xx_4.3.0_3.0_1672774178039.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_finetuned_for_xqua_on_chaii_xx_4.3.0_3.0_1672774178039.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_finetuned_for_xqua_on_chaii","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_finetuned_for_xqua_on_chaii","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_finetuned_for_xqua_on_chaii| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/distilbert-multi-finetuned-for-xqua-on-chaii --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from nbroad) author: John Snow Labs name: roberta_qa_base_super_2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rob-base-superqa2` is a English model originally trained by `nbroad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_super_2_en_4.3.0_3.0_1674212395438.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_super_2_en_4.3.0_3.0_1674212395438.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_super_2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_super_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_super_2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nbroad/rob-base-superqa2 - https://paperswithcode.com/sota?task=Question+Answering&dataset=squad_v2 --- layout: model title: Pipeline to Detect Clinical Entities in Romanian (Bert, Base, Cased) author: John Snow Labs name: ner_clinical_bert_pipeline date: 2023-03-09 tags: [licensed, clinical, ro, ner, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_clinical_bert](https://nlp.johnsnowlabs.com/2022/11/22/ner_clinical_bert_ro.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_pipeline_ro_4.3.0_3.2_1678352766945.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_pipeline_ro_4.3.0_3.2_1678352766945.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_clinical_bert_pipeline", "ro", "clinical/models") text = '''Solicitare: Angio CT cardio-toracic Dg. de trimitere Atrezie de valva pulmonara. Hipoplazie VS. Atrezie VAV stang. Anastomoza Glenn. Sp. Tromboza la nivelul anastomozei. Trimis de: Sectia Clinica Cardiologie (dr. Sue T.) Procedura Aparat GE Revolution HD. Branula albastra montata la nivelul membrului superior drept. Scout. Se administreaza 30 ml Iomeron 350 cu flux 2.2 ml/s, urmate de 20 ml ser fiziologic cu acelasi flux. Se efectueaza o examinare angio-CT cardiotoracica cu achizitii secventiale prospective la o frecventa cardiaca medie de 100/min.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_clinical_bert_pipeline", "ro", "clinical/models") val text = "Solicitare: Angio CT cardio-toracic Dg. de trimitere Atrezie de valva pulmonara. Hipoplazie VS. Atrezie VAV stang. Anastomoza Glenn. Sp. Tromboza la nivelul anastomozei. Trimis de: Sectia Clinica Cardiologie (dr. Sue T.) Procedura Aparat GE Revolution HD. Branula albastra montata la nivelul membrului superior drept. Scout. Se administreaza 30 ml Iomeron 350 cu flux 2.2 ml/s, urmate de 20 ml ser fiziologic cu acelasi flux. Se efectueaza o examinare angio-CT cardiotoracica cu achizitii secventiale prospective la o frecventa cardiaca medie de 100/min." val result = pipeline.fullAnnotate(text) ```

## Results ```bass | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------------------------|--------:|------:|:--------------------------|-------------:| | 0 | Angio CT | 12 | 19 | Imaging_Test | 0.96415 | | 1 | cardio-toracic | 21 | 34 | Body_Part | 0.4776 | | 2 | Atrezie | 53 | 59 | Disease_Syndrome_Disorder | 0.9602 | | 3 | valva pulmonara | 64 | 78 | Body_Part | 0.73105 | | 4 | Hipoplazie | 81 | 90 | Disease_Syndrome_Disorder | 0.628 | | 5 | VS | 92 | 93 | Body_Part | 0.9543 | | 6 | Atrezie | 96 | 102 | Disease_Syndrome_Disorder | 0.8763 | | 7 | VAV stang | 104 | 112 | Body_Part | 0.9444 | | 8 | Anastomoza Glenn | 115 | 130 | Disease_Syndrome_Disorder | 0.8648 | | 9 | Tromboza | 137 | 144 | Disease_Syndrome_Disorder | 0.991 | | 10 | GE Revolution HD | 238 | 253 | Medical_Device | 0.668367 | | 11 | Branula albastra | 256 | 271 | Medical_Device | 0.9179 | | 12 | membrului superior | 292 | 309 | Body_Part | 0.98815 | | 13 | drept | 311 | 315 | Direction | 0.5645 | | 14 | Scout | 318 | 322 | Body_Part | 0.3484 | | 15 | 30 ml | 342 | 346 | Dosage | 0.9996 | | 16 | Iomeron 350 | 348 | 358 | Drug_Ingredient | 0.9822 | | 17 | 2.2 ml/s | 368 | 375 | Dosage | 0.9327 | | 18 | 20 ml | 388 | 392 | Dosage | 0.9977 | | 19 | ser fiziologic | 394 | 407 | Drug_Ingredient | 0.9609 | | 20 | angio-CT | 452 | 459 | Imaging_Test | 0.9965 | | 21 | cardiotoracica | 461 | 474 | Body_Part | 0.9344 | | 22 | achizitii secventiale prospective | 479 | 511 | Imaging_Technique | 0.966833 | | 23 | 100/min | 546 | 552 | Pulse | 0.9128 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_bert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|ro| |Size:|483.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal Subrogation Clause Binary Classifier author: John Snow Labs name: legclf_subrogation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `subrogation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `subrogation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subrogation_clause_en_1.0.0_3.2_1660123050748.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subrogation_clause_en_1.0.0_3.2_1660123050748.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_subrogation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[subrogation]| |[other]| |[other]| |[subrogation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_subrogation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.92 0.97 0.95 89 subrogation 0.90 0.79 0.84 33 accuracy - - 0.92 122 macro-avg 0.91 0.88 0.89 122 weighted-avg 0.92 0.92 0.92 122 ``` --- layout: model title: German Public Health Mention Sequence Classifier (GBERT-large) author: John Snow Labs name: bert_sequence_classifier_health_mentions_gbert_large date: 2022-08-10 tags: [public_health, de, licensed, sequence_classification, health_mention] task: Text Classification language: de edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [GBERT-large](https://arxiv.org/pdf/2010.10906.pdf) based sequence classification model that can classify public health mentions in German social media text. ## Predicted Entities `non-health`, `health-related` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_health_mentions_gbert_large_de_4.0.2_3.0_1660133721898.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_health_mentions_gbert_large_de_4.0.2_3.0_1660133721898.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------------------------------------------------------+----------------+ |text |result | +---------------------------------------------------------------------------------------------------+----------------+ |Durch jahrelanges Rauchen habe ich meine Lunge einfach zu sehr geschädigt - Punkt. |[health-related]| |die Schatzsuche war das Highlight beim Kindergeburtstag, die kids haben noch lange davon gesprochen|[non-health] | +---------------------------------------------------------------------------------------------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_health_mentions_gbert_large| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|de| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References Curated from several academic and in-house datasets. ## Benchmarking ```bash label precision recall f1-score support non-health 0.99 0.99 0.99 82 health-related 0.99 0.99 0.99 69 accuracy - - 0.99 151 macro-avg 0.99 0.99 0.99 151 weighted-avg 0.99 0.99 0.99 151 ``` --- layout: model title: Legal Free Movement Of Capital Document Classifier (EURLEX) author: John Snow Labs name: legclf_free_movement_of_capital_bert date: 2023-03-06 tags: [en, legal, classification, clauses, free_movement_of_capital, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_free_movement_of_capital_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Free_Movement_of_Capital or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Free_Movement_of_Capital`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_free_movement_of_capital_bert_en_1.0.0_3.0_1678111740710.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_free_movement_of_capital_bert_en_1.0.0_3.0_1678111740710.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_free_movement_of_capital_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Free_Movement_of_Capital]| |[Other]| |[Other]| |[Free_Movement_of_Capital]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_free_movement_of_capital_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Free_Movement_of_Capital 0.85 0.93 0.89 67 Other 0.91 0.82 0.86 61 accuracy - - 0.88 128 macro-avg 0.88 0.87 0.87 128 weighted-avg 0.88 0.88 0.87 128 ``` --- layout: model title: Legal Subsidiaries Clause Binary Classifier author: John Snow Labs name: legclf_subsidiaries_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `subsidiaries` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `subsidiaries` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_subsidiaries_clause_en_1.0.0_3.2_1660124029869.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_subsidiaries_clause_en_1.0.0_3.2_1660124029869.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_subsidiaries_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[subsidiaries]| |[other]| |[other]| |[subsidiaries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_subsidiaries_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.95 0.97 63 subsidiaries 0.88 0.96 0.92 23 accuracy - - 0.95 86 macro-avg 0.93 0.95 0.94 86 weighted-avg 0.96 0.95 0.95 86 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_el6 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-el6` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el6_en_4.3.0_3.0_1675111427354.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_el6_en_4.3.0_3.0_1675111427354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_el6","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_el6","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_el6| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|393.2 MB| ## References - https://huggingface.co/google/t5-efficient-base-el6 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Legal Information And Information Processing Document Classifier (EURLEX) author: John Snow Labs name: legclf_information_and_information_processing_bert date: 2023-03-06 tags: [en, legal, classification, clauses, information_and_information_processing, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_information_and_information_processing_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Information_and_Information_Processing or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Information_and_Information_Processing`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_information_and_information_processing_bert_en_1.0.0_3.0_1678111806432.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_information_and_information_processing_bert_en_1.0.0_3.0_1678111806432.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_information_and_information_processing_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Information_and_Information_Processing]| |[Other]| |[Other]| |[Information_and_Information_Processing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_information_and_information_processing_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Information_and_Information_Processing 0.79 0.96 0.87 47 Other 0.93 0.70 0.80 40 accuracy - - 0.84 87 macro-avg 0.86 0.83 0.83 87 weighted-avg 0.86 0.84 0.84 87 ``` --- layout: model title: Detect Anatomical Structures in Medical Text author: John Snow Labs name: bert_token_classifier_ner_anatem date: 2022-07-25 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Anatomical entities ranging from subcellular structures to organ systems are central to biomedical science, and mentions of these entities are essential to understanding the scientific literature. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. The model detects anatomical structures from a medical text. ## Predicted Entities `Anatomy` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatem_en_4.0.0_3.0_1658749376934.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatem_en_4.0.0_3.0_1658749376934.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------+-------+ |ner_chunk |label | +----------------------+-------+ |Malignant cells |Anatomy| |cytoplasmic organelles|Anatomy| |tumor |Anatomy| |breast |Anatomy| |ovarian |Anatomy| |prostate cancer |Anatomy| +----------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_anatem| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References [https://github.com/cambridgeltl/MTL-Bioinformatics-2016](https://github.com/cambridgeltl/MTL-Bioinformatics-2016) ## Benchmarking ```bash label precision recall f1-score support B-Anatomy 0.8489 0.9380 0.8912 4616 I-Anatomy 0.9190 0.8839 0.9011 3247 micro-avg 0.8755 0.9157 0.8951 7863 macro-avg 0.8839 0.9110 0.8962 7863 weighted-avg 0.8778 0.9157 0.8953 7863 ``` --- layout: model title: Multilingual T5ForConditionalGeneration Base Cased model (from sonoisa) author: John Snow Labs name: t5_base_english_japanese date: 2023-01-30 tags: [en, ja, multilingual, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-english-japanese` is a Multilingual model originally trained by `sonoisa`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_english_japanese_xx_4.3.0_3.0_1675108659585.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_english_japanese_xx_4.3.0_3.0_1675108659585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_english_japanese","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_english_japanese","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_english_japanese| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|521.2 MB| ## References - https://huggingface.co/sonoisa/t5-base-english-japanese - https://en.wikipedia.org - https://ja.wikipedia.org - https://oscar-corpus.com - http://data.statmt.org/cc-100/ - http://data.statmt.org/cc-100/ - https://github.com/sonoisa/t5-japanese - https://creativecommons.org/licenses/by-sa/4.0/deed.ja - http://commoncrawl.org/terms-of-use/ --- layout: model title: Translate Kirundi to English Pipeline author: John Snow Labs name: translate_rn_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, rn, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `rn` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_rn_en_xx_2.7.0_2.4_1609688989566.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_rn_en_xx_2.7.0_2.4_1609688989566.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_rn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_rn_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.rn.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_rn_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal NER for NDA (Applicable Law Clause) author: John Snow Labs name: legner_nda_applicable_law date: 2023-04-06 tags: [en, licensed, legal, ner, nda, law] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `APPLIC_LAW ` clause with a proper classifier (use legmulticlf_mnda_sections_paragraph_other for that purpose). It will extract the following entities: `LAW `, and `LAW_LOCATION`. ## Predicted Entities `LAW`, `LAW_LOCATION` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_applicable_law_en_1.0.0_3.0_1680819415977.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_applicable_law_en_1.0.0_3.0_1680819415977.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_applicable_law", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""This Agreement will be governed and construed in accordance with the laws of the State of Utah without regard to the conflicts of laws or principles thereof."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-------------+------------+ |chunk |ner_label | +-------------+------------+ |laws |LAW | |State of Utah|LAW_LOCATION| |laws |LAW | +-------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_applicable_law| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support LAW 0.97 0.97 0.97 70 LAW_LOCATION 0.92 0.94 0.93 51 micro-avg 0.95 0.96 0.95 121 macro-avg 0.95 0.96 0.95 121 weighted-avg 0.95 0.96 0.95 121 ``` --- layout: model title: Chinese T5ForConditionalGeneration Cased model (from hululuzhu) author: John Snow Labs name: t5_chinese_couplet_mengzi_finetune date: 2023-01-30 tags: [zh, open_source, t5, tensorflow] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-couplet-t5-mengzi-finetune` is a Chinese model originally trained by `hululuzhu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_chinese_couplet_mengzi_finetune_zh_4.3.0_3.0_1675100437975.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_chinese_couplet_mengzi_finetune_zh_4.3.0_3.0_1675100437975.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_chinese_couplet_mengzi_finetune","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_chinese_couplet_mengzi_finetune","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_chinese_couplet_mengzi_finetune| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|1.0 GB| ## References - https://huggingface.co/hululuzhu/chinese-couplet-t5-mengzi-finetune - https://github.com/hululuzhu/chinese-ai-writing-share - https://github.com/hululuzhu/chinese-ai-writing-share/tree/main/slides - https://github.com/wb14123/couplet-dataset --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from BruceZJC) author: John Snow Labs name: distilbert_qa_brucezjc_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `BruceZJC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_brucezjc_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768353044.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_brucezjc_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768353044.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_brucezjc_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_brucezjc_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_brucezjc_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/BruceZJC/distilbert-base-uncased-finetuned-squad --- layout: model title: Sentence Entity Resolver for RxCUI (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxcui date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxCUI codes codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts RxCUI Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxcui_en_3.0.4_3.0_1621189488426.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxcui_en_3.0.4_3.0_1621189488426.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_rxcui``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_posology``` as NER model. ```DRUG``` set in ```.setWhiteList()```.

## Results ```bash +---------------------------+--------+-----------------------------------------------------+ | chunk | code | term | +---------------------------+--------+-----------------------------------------------------+ | 50 mg of eltrombopag oral | 825427 | eltrombopag 50 MG Oral Tablet | | 5 mg amlodipine | 197361 | amlodipine 5 MG Oral Tablet | | metformin 1000 mg | 861004 | metformin hydrochloride 2000 MG Oral Tablet | +---------------------------+--------+-----------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxcui| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxcui_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on November 2020 RxNorm Clinical Drugs ontology graph with ``sbiobert_base_cased_mli`` embeddings. https://www.nlm.nih.gov/pubs/techbull/nd20/brief/nd20_rx_norm_november_release.html. [Sample Content](https://rxnav.nlm.nih.gov/REST/rxclass/class/byRxcui.json?rxcui=1000000). --- layout: model title: English image_classifier_vit_dog_races ViTForImageClassification from roschmid author: John Snow Labs name: image_classifier_vit_dog_races date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_dog_races` is a English model originally trained by roschmid. ## Predicted Entities `Rottweiler`, `Shiba Inu`, `Chow Chow dog`, `Siberian Husky`, `German Shepherd`, `Pug`, `Border Collie`, `Golden Retriever`, `Tibetan Mastiff` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_races_en_4.1.0_3.0_1660168522778.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_dog_races_en_4.1.0_3.0_1660168522778.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_dog_races", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_dog_races", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_dog_races| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Part of Speech for Norwegian author: John Snow Labs name: pos_ud_bokmaal date: 2020-05-05 18:57:00 +0800 task: Part of Speech Tagging language: nb edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, nb] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bokmaal_nb_2.5.0_2.4_1588693881973.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bokmaal_nb_2.5.0_2.4_1588693881973.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_bokmaal", "nb") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_bokmaal", "nb") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Annet enn å være kongen i nord, er John Snow en engelsk lege og en leder innen utvikling av anestesi og medisinsk hygiene."""] pos_df = nlu.load('nb.pos.ud_bokmaal').predict(text) pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=4, result='DET', metadata={'word': 'Annet'}), Row(annotatorType='pos', begin=6, end=8, result='SCONJ', metadata={'word': 'enn'}), Row(annotatorType='pos', begin=10, end=10, result='PART', metadata={'word': 'å'}), Row(annotatorType='pos', begin=12, end=15, result='AUX', metadata={'word': 'være'}), Row(annotatorType='pos', begin=17, end=22, result='NOUN', metadata={'word': 'kongen'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bokmaal| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|nb| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Identify Competing Organizations author: John Snow Labs name: finassertion_competitors date: 2022-08-17 tags: [en, finance, assertion, status, competitors, licensed] task: Assertion Status language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This models allows you to identify ORG and PRODUCTS mentioned in the text to be from a competitor. By default, if nothing is mentioned, it returns `NO_COMPETITOR`. ## Predicted Entities `NO_COMPETITOR`, `COMPETITOR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/ASSERTIONDL_COMPETITORS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finassertion_competitors_en_1.0.0_3.2_1660735220316.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finassertion_competitors_en_1.0.0_3.2_1660735220316.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python # Annotator that transforms a text column from dataframe into an Annotation ready for NLP from johnsnowlabs import * documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") # Sentence Detector annotator, processes various sentences per line sentenceDetector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") # nlp.Tokenizer splits words in a relevant format for NLP tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model_org = finance.NerModel.pretrained("finner_orgs_prods_alias", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = finance.NerConverterInternal() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ .setWhiteList(['ORG', 'PRODUCT']) assertion = finance.AssertionDLModel.pretrained("finassertion_competitors", "en", "finance/models")\ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, ner_model_org, ner_converter, assertion ]) text = "Our competitors include the following by general category: legacy antivirus product providers, such as McAfee LLC and Broadcom Inc." data = spark.createDataFrame([[text]]).toDF("text") model = nlpPipeline.fit(data) model.transform(spark.createDataFrame([[text]]).toDF("text")).select(F.explode(F.arrays_zip('ner_chunk.result', 'assertion.result')).alias('result')).show(truncate=False) ```

## Results ```bash +--------------------------+ |result | +--------------------------+ |[McAfee LLC, COMPETITOR] | |[Broadcom Inc, COMPETITOR]| +--------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finassertion_competitors| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, doc_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|2.2 MB| ## References In-house annotations from 10K Filings ## Benchmarking ```bash label tp fp fn prec rec f1 NO_COMPETITOR 158 0 1 1.0 0.9937107 0.9968454 COMPETITOR 25 1 0 0.9615384 1.0 0.9803921 Macro-average 183 1 1 0.9807692 0.9968554 0.9887469 Micro-average 183 1 1 0.9945652 0.9945652 0.9945652 ``` --- layout: model title: Relation extraction between body parts and problem entities (ReDL) author: John Snow Labs name: redl_bodypart_problem_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts and problem entities in clinical texts. `1` : Shows that there is a relation between body part entity and entities labeled as problem ( diagnosis, symptom etc.), `0` : Shows that there no relation between body part and problem entities. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_problem_biobert_en_2.7.3_2.4_1612446369008.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_problem_biobert_en_2.7.3_2.4_1612446369008.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | chunk1 | entity2 | chunk2 | confidence | |---:|-----------:|:----------|:---------|:-----------------------------|:---------|-------------:| | 0 | 1 | Symptom | numbness | External_body_part_or_region | hand | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_bodypart_problem_biobert| |Compatibility:|Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Data Source Trained on internal dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.762 0.814 0.787 315 1 0.938 0.917 0.927 885 Avg. 0.850 0.865 0.857 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from habib1030) author: John Snow Labs name: distilbert_qa_habib1030_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `habib1030`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_habib1030_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770984625.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_habib1030_base_uncased_finetuned_squad_en_4.3.0_3.0_1672770984625.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_habib1030_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_habib1030_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_habib1030_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/habib1030/distilbert-base-uncased-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from West Slavic languages to English author: John Snow Labs name: opus_mt_zlw_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, zlw, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `zlw` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_zlw_en_xx_2.7.0_2.4_1609166809467.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_zlw_en_xx_2.7.0_2.4_1609166809467.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_zlw_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from aszidon) Custom Version-5 author: John Snow Labs name: distilbert_qa_custom5 date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbertcustom5` is a English model originally trained by `aszidon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom5_en_4.0.0_3.0_1654728059999.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_custom5_en_4.0.0_3.0_1654728059999.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom5","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_custom5","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.custom5.by_aszidon").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_custom5| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/aszidon/distilbertcustom5 --- layout: model title: English DistilBertForQuestionAnswering model (from deepakvk) author: John Snow Labs name: distilbert_qa_base_uncased_distilled_squad_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-distilled-squad-finetuned-squad` is a English model originally trained by `deepakvk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_distilled_squad_finetuned_squad_en_4.0.0_3.0_1654723814436.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_distilled_squad_finetuned_squad_en_4.0.0_3.0_1654723814436.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_distilled_squad_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_distilled_squad_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_deepakvk").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_distilled_squad_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/deepakvk/distilbert-base-uncased-distilled-squad-finetuned-squad --- layout: model title: Danish XLMRobertaForTokenClassification Cased model (from saattrupdan) author: John Snow Labs name: xlmroberta_ner_employment_contract_ner date: 2022-08-01 tags: [da, open_source, xlm_roberta, ner] task: Named Entity Recognition language: da edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `employment-contract-ner-da` is a Danish model originally trained by `saattrupdan`. ## Predicted Entities `SALARY`, `STARTDATE`, `WORKHOURS`, `WORKPLACE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_employment_contract_ner_da_4.1.0_3.0_1659353422793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_employment_contract_ner_da_4.1.0_3.0_1659353422793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_employment_contract_ner","da") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_employment_contract_ner","da") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_employment_contract_ner| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|da| |Size:|798.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/saattrupdan/employment-contract-ner-da --- layout: model title: French DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_fr_cased date: 2022-04-12 tags: [distilbert, embeddings, fr, open_source] task: Embeddings language: fr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-fr-cased` is a French model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_fr_cased_fr_3.4.2_3.0_1649783659773.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_fr_cased_fr_3.4.2_3.0_1649783659773.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_fr_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fr| |Size:|231.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-fr-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Clinical Deidentification (glove) author: John Snow Labs name: clinical_deidentification_glove date: 2022-03-04 tags: [deidentification, en, licensed, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with lightweight `glove_100d` embeddings and can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_en_3.4.1_3.0_1646402880942.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_en_3.4.1_3.0_1646402880942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification_glove", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification_glove","en","clinical/models") val result = deid_pipeline.annotate("Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.glove_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

## Results ```bash Masked with entity labels ------------------------------ Name : , Record date: , # . Dr. , ID: , IP . He is a male was admitted to the for cystectomy on . Patient's VIN : , SSN , Driver's license . Phone , , , E-MAIL: . Masked with chars ------------------------------ Name : [**************], Record date: [********], # [****]. Dr. [********], ID: [********], IP [************]. He is a [*********] male was admitted to the [**********] for cystectomy on [******]. Patient's VIN : [***************], SSN [**********], Driver's license [*********]. Phone [************], [***************], [***********], E-MAIL: [*************]. Masked with fixed length chars ------------------------------ Name : ****, Record date: ****, # ****. Dr. ****, ID: ****, IP ****. He is a **** male was admitted to the **** for cystectomy on ****. Patient's VIN : ****, SSN ****, Driver's license ****. Phone ****, ****, ****, E-MAIL: ****. Obfuscated ------------------------------ Name : Berneta Anis, Record date: 2093-02-19, # U4660137. Dr. Dr Worley Colonel, ID: ZJ:9570208, IP 005.005.005.005. He is a 67 male was admitted to the ST. LUKE'S HOSPITAL AT THE VINTAGE for cystectomy on 06-02-1981. Patient's VIN : 3CCCC22DDDD333888, SSN SSN-618-77-1042, Driver's license W693817528998. Phone 0496 46 46 70, 3100 weston rd, Shattuck, E-MAIL: Freddi@hotmail.com. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification_glove| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|181.3 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Extract Biomarkers and their Results author: John Snow Labs name: ner_oncology_biomarker_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, ner, biomarker] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of biomarkers and biomarker results from oncology texts. Definitions of Predicted Entities: - `Biomarker`: Biological molecules that indicate the presence or absence of cancer, or the type of cancer (including oncogenes). - `Biomarker_Result`: Terms or values that are identified as the result of a biomarkers. ## Predicted Entities `Biomarker`, `Biomarker_Result` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_wip_en_4.0.0_3.0_1664584581032.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_wip_en_4.0.0_3.0_1664584581032.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------------------------------|:-----------------| | negative | Biomarker_Result | | CK7 | Biomarker | | synaptophysin | Biomarker | | Syn | Biomarker | | chromogranin A | Biomarker | | CgA | Biomarker | | Muc5AC | Biomarker | | human epidermal growth factor receptor-2 | Biomarker | | HER2 | Biomarker | | Muc6 | Biomarker | | positive | Biomarker_Result | | CK20 | Biomarker | | Muc1 | Biomarker | | Muc2 | Biomarker | | E-cadherin | Biomarker | | p53 | Biomarker | | Ki-67 index | Biomarker | | 87% | Biomarker_Result | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_biomarker_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|841.8 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Biomarker_Result 877.0 205.0 229.0 1106.0 0.81 0.79 0.80 Biomarker 1305.0 166.0 163.0 1468.0 0.89 0.89 0.89 macro_avg 2182.0 371.0 392.0 2574.0 0.85 0.84 0.84 micro_avg NaN NaN NaN NaN 0.85 0.85 0.85 ``` --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Original_BioBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD-Original-BioBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Original_BioBERT_384_en_4.0.0_3.0_1657108919488.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Original_BioBERT_384_en_4.0.0_3.0_1657108919488.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Original_BioBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD-Original-BioBERT-384 --- layout: model title: English RobertaForQuestionAnswering Cased model (from billfrench) author: John Snow Labs name: roberta_qa_cyberlandr_door date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cyberlandr-door` is a English model originally trained by `billfrench`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cyberlandr_door_en_4.3.0_3.0_1674209416902.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cyberlandr_door_en_4.3.0_3.0_1674209416902.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cyberlandr_door","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cyberlandr_door","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cyberlandr_door| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|414.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/billfrench/cyberlandr-door --- layout: model title: Detect Entities in General Scope (Few-NERD dataset) author: John Snow Labs name: nerdl_fewnerd_100d date: 2021-07-02 tags: [ner, en, fewnerd, public, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.1.1 spark_version: 2.4 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Few-NERD/inter public dataset and it extracts 8 entities that are in general scope. ## Predicted Entities `PERSON`, `ORGANIZATION`, `LOCATION`, `ART`, `BUILDING`, `PRODUCT`, `EVENT`, `OTHER` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_en_3.1.1_2.4_1625227974733.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_100d_en_3.1.1_2.4_1625227974733.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use The model is trained using `glove_100d` word embeddings so, you should use the same embeddings in your nlp pipeline.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_100d", "en")\ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("nerdl_fewnerd_100d") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(['document', 'token', 'ner']) \ .setOutputCol('ner_chunk') nlp_pipeline = Pipeline(stages=[document_assembler, sentencer, tokenizer, embeddings, ner, ner_converter]) l_model = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = l_model.fullAnnotate("""The Double Down is a sandwich offered by Kentucky Fried Chicken (KFC) restaurants. He did not see active service again until 1882, when he took part in the Anglo-Egyptian War, and was present at the battle of Tell El Kebir (September 1882), for which he was mentioned in dispatches, received the Egypt Medal with clasp and the 3rd class of the Order of Medjidie, and was appointed a Companion of the Order of the Bath (CB).""") ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_100d", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("nerdl_fewnerd_100d") .setInputCols(Array("sentence", "token", "embeddings")).setOutputCol("ner") val ner_converter = NerConverter.setInputCols(Array("document", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, ner, ner_converter)) val data = Seq("The Double Down is a sandwich offered by Kentucky Fried Chicken (KFC) restaurants. He did not see active service again until 1882, when he took part in the Anglo-Egyptian War, and was present at the battle of Tell El Kebir (September 1882), for which he was mentioned in dispatches, received the Egypt Medal with clasp and the 3rd class of the Order of Medjidie, and was appointed a Companion of the Order of the Bath (CB).").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.fewnerd").predict("""The Double Down is a sandwich offered by Kentucky Fried Chicken (KFC) restaurants. He did not see active service again until 1882, when he took part in the Anglo-Egyptian War, and was present at the battle of Tell El Kebir (September 1882), for which he was mentioned in dispatches, received the Egypt Medal with clasp and the 3rd class of the Order of Medjidie, and was appointed a Companion of the Order of the Bath (CB).""") ```

## Results ```bash +----------------------------------+---------+ |chunk |ner_label| +----------------------------------+---------+ |Double Down |PRODUCT | |Kentucky Fried Chicken |BUILDING | |KFC |BUILDING | |Anglo-Egyptian War |EVENT | |Tell El Kebir |EVENT | |Egypt Medal |OTHER | |Order of Medjidie |OTHER | |Companion of the Order of the Bath|OTHER | +----------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_fewnerd_100d| |Type:|ner| |Compatibility:|Spark NLP 3.1.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Few-NERD:A Few-shot Named Entity Recognition Dataset, author: Ding, Ning and Xu, Guangwei and Chen, Yulin, and Wang, Xiaobin and Han, Xu and Xie, Pengjun and Zheng, Hai-Tao and Liu, Zhiyuan, book title: ACL-IJCNL, 2021. ## Benchmarking ```bash +------------+-------+------+-------+-------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +------------+-------+------+-------+-------+---------+------+------+ | PERSON|21555.0|6194.0| 5643.0|27198.0| 0.7768|0.7925|0.7846| |ORGANIZATION|36744.0|9059.0|13156.0|49900.0| 0.8022|0.7364|0.7679| | LOCATION|36367.0|7521.0| 7006.0|43373.0| 0.8286|0.8385|0.8335| | ART| 6170.0|1649.0| 2998.0| 9168.0| 0.7891| 0.673|0.7264| | BUILDING| 5112.0|2435.0| 3014.0| 8126.0| 0.6774|0.6291|0.6523| | PRODUCT| 8317.0|3253.0| 4325.0|12642.0| 0.7188|0.6579| 0.687| | OTHER|14461.0|4414.0| 5161.0|19622.0| 0.7661| 0.737|0.7513| | EVENT| 6024.0|1880.0| 2275.0| 8299.0| 0.7621|0.7259|0.7436| +------------+-------+------+-------+-------+---------+------+------+ +------------------+ | macro| +------------------+ |0.7433252741184967| +------------------+ +------------------+ | micro| +------------------+ |0.7703038245945377| +------------------+ ``` --- layout: model title: Stopwords Remover for Chinese language (1892 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, zh, open_source] task: Stop Words Removal language: zh edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_zh_3.4.1_3.0_1646673208417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_zh_3.4.1_3.0_1646673208417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------------------------------------------------+ |result | +-----------------------------------------------------------------------------------+ |[除了作为北方之王之外，约翰·斯诺还是一位英国医生，也是麻醉和医疗卫生发展的领导者。]| +-----------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|zh| |Size:|8.1 KB| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_10_h_768 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-10_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_768_zh_4.2.4_3.0_1670325770644.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_768_zh_4.2.4_3.0_1670325770644.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_10_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|330.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-10_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Detect PHI for Deidentification (Generic - Augmented) author: John Snow Labs name: ner_deid_generic_augmented date: 2021-06-30 tags: [clinical, deid, ner, en, licensed] task: [Named Entity Recognition, De-identification] language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN. Deidentification NER (Generic - Augmented) is a Named Entity Recognition model that annotates text to find protected health information that may need to be de-identified. It detects 7 entities. This ner model is trained with a combination of i2b2 train set and augmented version of i2b2 train set.## Predicted Entities`DATE`, `NAME`, `LOCATION`, `PROFESSION`, `CONTACT`, `AGE`, `ID`. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities `DATE`, `NAME`, `LOCATION`, `PROFESSION`, `CONTACT`, `AGE`, `ID` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_en_3.1.0_2.4_1625050266197.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_augmented_en_3.1.0_2.4_1625050266197.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") deid_ner = MedicalNerModel.pretrained("ner_deid_generic_augmented", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk_generic") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, deid_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame(pd.DataFrame({"text": ["""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227."""]}))) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val deid_ner = MedicalNerModel.pretrained("ner_deid_generic_augmented", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk_generic") val nlpPipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, deid_ner, ner_converter)) val data = Seq("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.generic_augmented").predict("""A. Record date : 2093-01-13, David Hale, M.D., Name : Hendrickson, Ora MR. # 7194334 Date : 01/13/93 PCP : Oliveira, 25 -year-old, Record date : 1-11-2000. Cocke County Baptist Hospital. 0295 Keats Street. Phone +1 (302) 786-5227.""") ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |2093-01-13 |DATE | |David Hale |NAME | |Hendrickson, Ora |NAME | |7194334 |ID | |01/13/93 |DATE | |Oliveira |NAME | |25 |AGE | |1-11-2000 |DATE | |Cocke County Baptist Hospital|LOCATION | |0295 Keats Street |LOCATION | |(302) 786-5227 |CONTACT | +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_augmented| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source A custom data set which is created from the i2b2-PHI train and the augmented version of the i2b2-PHI train set is used. ## Benchmarking ```bash entity tp fp fn total precision recall f1 CONTACT 341.0 15.0 14.0 355.0 0.9579 0.9606 0.9592 NAME 5065.0 165.0 205.0 5270.0 0.9685 0.9611 0.9648 DATE 5532.0 53.0 110.0 5642.0 0.9905 0.9805 0.9855 ID 614.0 23.0 71.0 685.0 0.9639 0.8964 0.9289 LOCATION 2686.0 164.0 284.0 2970.0 0.9425 0.9044 0.923 PROFESSION 228.0 28.0 145.0 373.0 0.8906 0.6113 0.725 AGE 713.0 34.0 49.0 762.0 0.9545 0.9357 0.945 macro - - - - - - 0.91876 micro - - - - - - 0.95616 ``` --- layout: model title: Ukrainian XlmRoBertaForQuestionAnswering model (from robinhad) author: John Snow Labs name: xlmroberta_qa_ukrainian date: 2022-06-28 tags: [uk, open_source, xlmroberta, question_answering] task: Question Answering language: uk edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ukrainian-qa` is a Ukrainian model originally trained by `robinhad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_qa_ukrainian_uk_4.0.0_3.0_1656418914986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_qa_ukrainian_uk_4.0.0_3.0_1656418914986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlmroberta_qa_ukrainian","uk") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Як мене звати?", "Мене звуть Клара, і я живу в Берклі."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = XlmRoBertaForQuestionAnswering.pretrained("xlmroberta_qa_ukrainian","uk") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Як мене звати?", "Мене звуть Клара, і я живу в Берклі.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("uk.answer_question.xlmr_roberta").predict("""Як мене звати?|||"Мене звуть Клара, і я живу в Берклі.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_qa_ukrainian| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|uk| |Size:|402.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/robinhad/ukrainian-qa - https://github.com/fido-ai/ua-datasets/tree/main/ua_datasets/src/question_answering - https://github.com/robinhad/ukrainian-qa --- layout: model title: Fast Neural Machine Translation Model from English to Romance Languages author: John Snow Labs name: opus_mt_en_roa date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, roa, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `roa` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_roa_xx_2.7.0_2.4_1609168782781.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_roa_xx_2.7.0_2.4_1609168782781.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_roa| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Sentence Detection in Yiddish Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [open_source, sentence_detection, yi] task: Sentence Detection language: yi edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_yi_3.2.0_3.0_1630323089681.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_yi_3.2.0_3.0_1630323089681.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "yi") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""איר זוכט פֿאַר אַ גרויס מקור פון לייענען פּאַראַגראַפס אין ענגליש? איר'ווע קומען צו די רעכט אָרט. לויט צו אַ פריש לערנען, די מידע פון לייענען אין הייַנט ס יוגנט איז ראַפּאַדלי דיקריסינג. זיי קענען נישט פאָקוס אויף אַ געגעבן פּאַראַגראַף פֿאַר ענגליש לייענען פֿאַר מער ווי אַ ביסל סעקונדעס! לייענען איז געווען און איז אַ ינטאַגראַל טייל פון אַלע קאַמפּעטיטיוו יגזאַמז. אַזוי ווי טאָן איר פֿאַרבעסערן דיין לייענען סקילז? דער ענטפער צו דעם קשיא איז אַקשלי אן אנדער קשיא: וואָס איז די נוצן פון לייענען סקילז? דער הויפּט ציל פון לייענען איז 'צו מאַכן זינען'.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "yi") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("איר זוכט פֿאַר אַ גרויס מקור פון לייענען פּאַראַגראַפס אין ענגליש? איר'ווע קומען צו די רעכט אָרט. לויט צו אַ פריש לערנען, די מידע פון לייענען אין הייַנט ס יוגנט איז ראַפּאַדלי דיקריסינג. זיי קענען נישט פאָקוס אויף אַ געגעבן פּאַראַגראַף פֿאַר ענגליש לייענען פֿאַר מער ווי אַ ביסל סעקונדעס! לייענען איז געווען און איז אַ ינטאַגראַל טייל פון אַלע קאַמפּעטיטיוו יגזאַמז. אַזוי ווי טאָן איר פֿאַרבעסערן דיין לייענען סקילז? דער ענטפער צו דעם קשיא איז אַקשלי אן אנדער קשיא: וואָס איז די נוצן פון לייענען סקילז? דער הויפּט ציל פון לייענען איז 'צו מאַכן זינען'.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('yi.sentence_detector').predict("איר זוכט פֿאַר אַ גרויס מקור פון לייענען פּאַראַגראַפס אין ענגליש? איר'ווע קומען צו די רעכט אָרט. לויט צו אַ פריש לערנען, די מידע פון לייענען אין הייַנט ס יוגנט איז ראַפּאַדלי דיקריסינג. זיי קענען נישט פאָקוס אויף אַ געגעבן פּאַראַגראַף פֿאַר ענגליש לייענען פֿאַר מער ווי אַ ביסל סעקונדעס! לייענען איז געווען און איז אַ ינטאַגראַל טייל פון אַלע קאַמפּעטיטיוו יגזאַמז. אַזוי ווי טאָן איר פֿאַרבעסערן דיין לייענען סקילז? דער ענטפער צו דעם קשיא איז אַקשלי אן אנדער קשיא: וואָס איז די נוצן פון לייענען סקילז? דער הויפּט ציל פון לייענען איז 'צו מאַכן זינען'.", output_level ='sentence') ```

## Results ```bash +--------------------------------------------------------------------------------------------------------+ |result | +--------------------------------------------------------------------------------------------------------+ |[איר זוכט פֿאַר אַ גרויס מקור פון לייענען פּאַראַגראַפס אין ענגליש?] | |[איר'ווע קומען צו די רעכט אָרט.] | |[לויט צו אַ פריש לערנען, די מידע פון לייענען אין הייַנט ס יוגנט איז ראַפּאַדלי דיקריסינג.] | |[זיי קענען נישט פאָקוס אויף אַ געגעבן פּאַראַגראַף פֿאַר ענגליש לייענען פֿאַר מער ווי אַ ביסל סעקונדעס!] | |[לייענען איז געווען און איז אַ ינטאַגראַל טייל פון אַלע קאַמפּעטיטיוו יגזאַמז.] | |[אַזוי ווי טאָן איר פֿאַרבעסערן דיין לייענען סקילז?] | |[דער ענטפער צו דעם קשיא איז אַקשלי אן אנדער קשיא:] | |[וואָס איז די נוצן פון לייענען סקילז?] | |[דער הויפּט ציל פון לייענען איז 'צו מאַכן זינען'.] | +--------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|yi| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: English Named Entity Recognition (from kleinay) author: John Snow Labs name: bert_ner_nominalization_candidate_classifier date: 2022-05-09 tags: [bert, ner, token_classification, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `nominalization-candidate-classifier` is a English model orginally trained by `kleinay`. ## Predicted Entities `False`, `True` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_nominalization_candidate_classifier_en_3.4.2_3.0_1652096794094.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_nominalization_candidate_classifier_en_3.4.2_3.0_1652096794094.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_nominalization_candidate_classifier| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/kleinay/nominalization-candidate-classifier - https://www.aclweb.org/anthology/2020.coling-main.274/ - https://github.com/kleinay/QANom --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_el32 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-el32` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el32_en_4.3.0_3.0_1675120001777.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_el32_en_4.3.0_3.0_1675120001777.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_el32","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_el32","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_el32| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|305.0 MB| ## References - https://huggingface.co/google/t5-efficient-small-el32 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English BertForQuestionAnswering model (from amoux) author: John Snow Labs name: bert_qa_scibert_nli_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `scibert_nli_squad` is a English model orginally trained by `amoux`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_nli_squad_en_4.0.0_3.0_1654189419720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_nli_squad_en_4.0.0_3.0_1654189419720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_scibert_nli_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_scibert_nli_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.scibert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_scibert_nli_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/amoux/scibert_nli_squad --- layout: model title: Financial Executives compensation Item Binary Classifier author: John Snow Labs name: finclf_executives_compensation_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `executives_compensation` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `executives_compensation` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_executives_compensation_item_en_1.0.0_3.2_1660154405348.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_executives_compensation_item_en_1.0.0_3.2_1660154405348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_executives_compensation_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[executives_compensation]| |[other]| |[other]| |[executives_compensation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_executives_compensation_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.5 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support executives_compensation 0.93 0.94 0.94 104 other 0.93 0.92 0.92 83 accuracy - - 0.93 187 macro-avg 0.93 0.93 0.93 187 weighted-avg 0.93 0.93 0.93 187 ``` --- layout: model title: Extract clinical problems (Voice of the Patients) - Granular version author: John Snow Labs name: ner_vop_problem_wip date: 2023-04-20 tags: [licensed, clinical, en, ner, vop, patient, problem] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts clinical problems from the documents transferred from the patient’s own sentences using a granular taxonomy. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `InjuryOrPoisoning`, `Modifier`, `HealthStatus`, `Symptom`, `Disease`, `PsychologicalCondition` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_wip_en_4.4.0_3.0_1682012767167.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_wip_en_4.4.0_3.0_1682012767167.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------------|:------------| | pain | Symptom | | fatigue | Symptom | | rheumatoid arthritis | Disease | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_problem_wip| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.9 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I”m 20 year old girl. I”m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I”m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I”m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I”m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 InjuryOrPoisoning 93 33 52 145 0.74 0.64 0.69 Modifier 695 210 292 987 0.77 0.70 0.73 HealthStatus 80 20 25 105 0.80 0.76 0.78 Symptom 3669 440 1016 4685 0.89 0.78 0.83 Disease 1708 261 262 1970 0.87 0.87 0.87 PsychologicalCondition 315 30 26 341 0.91 0.92 0.92 macro_avg 6560 994 1673 8233 0.83 0.78 0.80 micro_avg 6560 994 1673 8233 0.87 0.80 0.83 ``` --- layout: model title: US Financial News Articles Summarization(Headers) author: John Snow Labs name: finsum_us_news_headers date: 2023-03-04 tags: [en, licensed, finance, summarization, t5, tensorflow] task: Summarization language: en edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true recommended: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is fine-tuned with US news articles from `bloomberg.com`, `cnbc.com`, `reuters.com`, `wsj.com`, and `fortune.com`. It is a Financial News Summarizer, aimed to extract headers from US financial news. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finsum_us_news_headers_en_1.0.0_3.0_1677962901205.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finsum_us_news_headers_en_1.0.0_3.0_1677962901205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("documents") t5 = nlp.T5Transformer().pretrained("finsum_us_news_headers", "en", "finance/models") \ .setTask("summarization") \ .setInputCols(["documents"]) \ .setMaxOutputLength(512) \ .setOutputCol("summaries") data = spark.createDataFrame([["Kindred Systems Inc. was founded with the mission of creating human-like intelligence in machines and a vision to commercialize its research work in tandem. Over the past few years, Kindred’s Product and Artificial General Intelligence (AGI) divisions have accomplished a tremendous amount in their respective domains, working independently to allow each team to optimize for their objectives. The company has reached a point in its evolution where spinning off the AGI division maximizes the likelihood of success for both divisions, as well as returns to Kindred shareholders. Geordie Rose is stepping down as CEO and President of Kindred to lead this new entity named Sanctuary based in Vancouver, Canada. Kindred co-founder Suzanne Gildert will also be stepping down from her role as Chief Science Officer and will join Sanctuary as co-CEO. Sanctuary’s focus is on the implementation and testing of a specific framework for artificial general intelligence. The new entity will license some of Kindred’s patents and software, and Kindred will maintain a minority ownership in Sanctuary. Kindred’s Board of Directors has appointed Jim Liefer, previously the company’s COO, to serve as CEO and President. As COO, Liefer brought strong executive leadership alongside co-founder, George Babu, for the development and deployment of Kindred’s first commercial product Sort, and will continue to lead the company in its mission to research and develop human-like intelligence in machines. The Kindred team in Toronto will continue its applied research in machine and reinforcement learning, with the San Francisco office focused on robotics, product development, and commercialization. With Kindred Sort, the company aims to alleviate the massive pressures facing the retail and fulfillment industry, which includes significant online sales growth, labor shortages, and a lack of advancement in technology. Kindred Sort allows retailers to manage the exploding growth and demand of this sector more efficiently. During the 2017 holiday season, Kindred’s robots sorted thousands of items ordered at speeds averaging over 410 units per hour, and reaching peak speeds of over 531 units per hour, freeing human workers to perform other parts of the fulfillment process critical to meet growing customer demand. “Kindred will maintain its commitment to building human-like intelligence in machines and applying those learnings to create and teach a new intelligent class of robots that will enhance the quality of our day-to-day lives, and in particular, the way we work,” said Liefer. “We look forward to advancing Kindred Sort, achieving new AI and robotic milestones while also helping to drive retail and other industries forward.” Babu, Kindred co-founder, and Chief Product Officer will be joining Liefer on Kindred’s Board of Directors. Babu will continue to oversee product strategy and the expansion of Kindred’s partnerships and pilot programs with major global retailers. Kindred Systems Inc.’s mission is to build machines with human-like intelligence. The company’s central thesis is that intelligence requires a body. Since its founding in 2014, Kindred has been exploring and engineering systems that enable robots to understand and participate in our world, with the ultimate goal of a future where intelligent machines work together with people. Kindred is headquartered in San Francisco with an office in Toronto."]]).toDF("text") pipeline = nlp.Pipeline().setStages([document_assembler, t5]) results = pipeline.fit(data).transform(data) ```

## Results ```bash Kindred Systems Inc. Announces Appointment of Jim Liefer as CEO and President ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finsum_us_news_headers| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[summaries]| |Language:|en| |Size:|925.8 MB| ## References Train dataset available [here](https://www.kaggle.com/datasets/jeet2016/us-financial-news-articles) --- layout: model title: Legal Benefits Clause Binary Classifier author: John Snow Labs name: legclf_benefits_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `benefits` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `benefits` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_benefits_clause_en_1.0.0_3.2_1660123261684.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_benefits_clause_en_1.0.0_3.2_1660123261684.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_benefits_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[benefits]| |[other]| |[other]| |[benefits]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_benefits_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support benefits 0.92 0.92 0.92 25 other 0.98 0.98 0.98 89 accuracy - - 0.96 114 macro-avg 0.95 0.95 0.95 114 weighted-avg 0.96 0.96 0.96 114 ``` --- layout: model title: Bangla RobertaForMaskedLM Cased model (from neuralspace-reverie) author: John Snow Labs name: roberta_embeddings_indic_transformers date: 2022-12-12 tags: [bn, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: bn edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `indic-transformers-bn-roberta` is a Bangla model originally trained by `neuralspace-reverie`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_bn_4.2.4_3.0_1670858623995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_indic_transformers_bn_4.2.4_3.0_1670858623995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indic_transformers","bn") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_indic_transformers","bn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_indic_transformers| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|bn| |Size:|312.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/neuralspace-reverie/indic-transformers-bn-roberta - https://oscar-corpus.com/ --- layout: model title: Extract Tickers on Financial Texts (RoBerta) author: John Snow Labs name: finner_roberta_ticker date: 2022-08-09 tags: [en, financial, ner, ticker, trading, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model aims to detect Trading Symbols / Tickers in texts. You can then use Chunk Mappers to get more information about the company that ticker belongs to. This is a RoBerta-based model, you can find other lighter versions of this model in Models Hub. ## Predicted Entities `TICKER` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_TICKER/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_roberta_ticker_en_1.0.0_3.2_1660036613729.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_roberta_ticker_en_1.0.0_3.2_1660036613729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['AMZN'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_roberta_ticker| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|465.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References Original dataset (https://www.kaggle.com/omermetinn/tweets-about-the-top-companies-from-2015-to-2020) and weak labelling on in-house texts ## Benchmarking ```bash label precision recall f1-score support TICKER 0.98 0.97 0.98 9823 micro-avg 0.98 0.97 0.98 9823 macro-avg 0.98 0.97 0.98 9823 weighted-avg 0.98 0.97 0.98 9823 ``` --- layout: model title: Legal Business Classification Document Classifier (EURLEX) author: John Snow Labs name: legclf_business_classification_bert date: 2023-03-06 tags: [en, legal, classification, clauses, business_classification, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_business_classification_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Business_Classification or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Business_Classification`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_business_classification_bert_en_1.0.0_3.0_1678111559659.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_business_classification_bert_en_1.0.0_3.0_1678111559659.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_business_classification_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Business_Classification]| |[Other]| |[Other]| |[Business_Classification]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_business_classification_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Business_Classification 0.81 0.94 0.87 32 Other 0.92 0.76 0.83 29 accuracy - - 0.85 61 macro-avg 0.86 0.85 0.85 61 weighted-avg 0.86 0.85 0.85 61 ``` --- layout: model title: Fast Neural Machine Translation Model from American Sign Language to English author: John Snow Labs name: opus_mt_ase_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ase, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ase` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ase_en_xx_2.7.0_2.4_1609169701112.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ase_en_xx_2.7.0_2.4_1609169701112.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ase_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Extract relations between NIHSS entities author: John Snow Labs name: redl_nihss_biobert date: 2023-01-15 tags: [en, licensed, clinical, relation_extraction, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relate scale items and their measurements according to NIHSS guidelines. ## Predicted Entities `Has_Value`, `0` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_NIHSS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_nihss_biobert_en_4.2.4_3.0_1673762755276.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_nihss_biobert_en_4.2.4_3.0_1673762755276.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+-----------+-------------+-----------+-----------+------------+-------------+-----------+--------------------+----------+ | relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +---------+-----------+-------------+-----------+-----------+------------+-------------+-----------+--------------------+----------+ |Has_Value| NIHSS| 20| 30|NIHSS score| Measurement| 36| 36| 4| 0.9998851| |Has_Value|Measurement| 89| 89| 2| 6a_LeftLeg| 111| 118| left leg| 0.9987311| | 0|Measurement| 89| 89| 2| Measurement| 124| 124| 2|0.97510725| | 0|Measurement| 89| 89| 2|7_LimbAtaxia| 156| 185|ataxia in her lef...| 0.999889| | 0| 6a_LeftLeg| 111| 118| left leg| Measurement| 124| 124| 2|0.99989617| | 0| 6a_LeftLeg| 111| 118| left leg|7_LimbAtaxia| 156| 185|ataxia in her lef...| 0.9999521| |Has_Value|Measurement| 124| 124| 2|7_LimbAtaxia| 156| 185|ataxia in her lef...| 0.9896319| +---------+-----------+-------------+-----------+-----------+------------+-------------+-----------+--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_nihss_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References @article{wangnational, title={National Institutes of Health Stroke Scale (NIHSS) Annotations for the MIMIC-III Database}, author={Wang, Jiayang and Huang, Xiaoshuo and Yang, Lin and Li, Jiao} } ## Benchmarking ```bash label Recall Precision F1 Support 0 0.989 0.976 0.982 611 Has_Value 0.983 0.992 0.988 889 Avg. 0.986 0.984 0.985 - Weighted-Avg. 0.985 0.985 0.985 - ``` --- layout: model title: Clinical Deidentification (English, Glove, Augmented) author: John Snow Labs name: clinical_deidentification_glove_augmented date: 2022-06-30 tags: [en, deid, deidentification, clinical, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline is trained with lightweight `glove_100d` embeddings and can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR` entities. It's different to `clinical_deidentification_glove` in the way it manages PHONE and PATIENT, having apart from the NER, some rules in Contextual Parser components. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_4.0.0_3.0_1656579032191.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_glove_augmented_en_4.0.0_3.0_1656579032191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification_glove_augmented", "en", "clinical/models") deid_pipeline.annotate("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = PretrainedPipeline("clinical_deidentification_glove_augmented","en","clinical/models") val result = pipeline.annotate("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.deid.glove_augmented.pipeline").predict("""Record date : 2093-01-13, David Hale, M.D. IP: 203.120.223.13. The driver's license no:A334455B. the SSN: 324598674 and e-mail: hale@gmail.com. Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93. PCP : Oliveira, 25 years old. Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286.""") ```

## Results ```bash {'masked': ['Record date : , , M.D.', 'IP: .', "The driver's license no: .", 'The SSN: and e-mail: .', 'Name : MR. # Date : .', 'PCP : , years old.', 'Record date : , : .'], 'masked_fixed_length_chars': ['Record date : ****, ****, M.D.', 'IP: ****.', "The driver's license no: ****.", 'The SSN: **** and e-mail: ****.', 'Name : **** MR. # **** Date : ****.', 'PCP : ****, **** years old.', 'Record date : ****, **** : ****.'], 'masked_with_chars': ['Record date : [********], [********], M.D.', 'IP: [************].', "The driver's license no: [******].", 'The SSN: [*******] and e-mail: [************].', 'Name : [**************] MR. # [****] Date : [******].', 'PCP : [******], ** years old.', 'Record date : [********], [***********] : [***************].'], 'ner_chunk': ['2093-01-13', 'David Hale', 'A334455B', '324598674', 'hale@gmail.com', 'Hendrickson, Ora', '719435', '01/13/93', 'Oliveira', '25', '2079-11-09', "Patient's VIN", '1HGBH41JXMN109286'], 'obfuscated': ['Record date : 2093-01-23, Dr Marshia Curling, M.D.', 'IP: 004.004.004.004.', "The driver's license no: 123XX123.", 'The SSN: SSN-089-89-9294 and e-mail: Mikey@hotmail.com.', 'Name : Stephania Chang MR. # E5881795 Date : 02-14-1983.', 'PCP : Dr Lovella Israel, 52 years old.', 'Record date : 2079-11-14, Dr Colie Carne : 3CCCC22DDDD333888.'], 'sentence': ['Record date : 2093-01-13, David Hale, M.D.', 'IP: 203.120.223.13.', "The driver's license no: A334455B.", 'The SSN: 324598674 and e-mail: hale@gmail.com.', 'Name : Hendrickson, Ora MR. # 719435 Date : 01/13/93.', 'PCP : Oliveira, 25 years old.', "Record date : 2079-11-09, Patient's VIN : 1HGBH41JXMN109286."]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification_glove_augmented| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|182.7 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - MedicalNerModel - NerConverterInternalModel - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Legal Loan and Security Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_loan_and_security_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, loan, security, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_loan_and_security_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `loan-and-security-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `loan-and-security-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_loan_and_security_agreement_en_1.0.0_3.0_1668109815627.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_loan_and_security_agreement_en_1.0.0_3.0_1668109815627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[loan-and-security-agreement]| |[other]| |[other]| |[loan-and-security-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_loan_and_security_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support loan-and-security-agreement 0.94 1.00 0.97 33 other 1.00 0.98 0.99 85 accuracy - - 0.98 118 macro-avg 0.97 0.99 0.98 118 weighted-avg 0.98 0.98 0.98 118 ``` --- layout: model title: English BertForQuestionAnswering Cased model (from Aiyshwariya) author: John Snow Labs name: bert_qa_aiyshwariya_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `Aiyshwariya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_aiyshwariya_finetuned_squad_en_4.0.0_3.0_1657185952130.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_aiyshwariya_finetuned_squad_en_4.0.0_3.0_1657185952130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_aiyshwariya_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_aiyshwariya_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_aiyshwariya_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Aiyshwariya/bert-finetuned-squad --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC4CHEMD_Imbalancedscibert_scivocab_cased date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC4CHEMD_Imbalancedscibert_scivocab_cased` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Imbalancedscibert_scivocab_cased_en_4.0.0_3.0_1657108990893.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC4CHEMD_Imbalancedscibert_scivocab_cased_en_4.0.0_3.0_1657108990893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC4CHEMD_Imbalancedscibert_scivocab_cased| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC4CHEMD_Imbalancedscibert_scivocab_cased --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_3 TFWav2Vec2ForCTC from doddle124578 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_3 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_3` is a English model originally trained by doddle124578. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_3_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_3_en_4.2.0_3.0_1664038287215.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_3_en_4.2.0_3.0_1664038287215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_3", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_3", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_3| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Detect concepts in drug development trials (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_drug_development_trials date: 2022-03-22 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description It is a `BertForTokenClassification` NER model to identify concepts related to drug development including `Trial Groups` , `End Points` , `Hazard Ratio` and other entities in free text. ## Predicted Entities `Patient_Count`, `Duration`, `End_Point`, `Value`, `Trial_Group`, `Hazard_Ratio`, `Total_Patients` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DRUGS_DEVELOPMENT_TRIALS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_BERT_TOKEN_CLASSIFIER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_drug_development_trials_en_3.3.4_3.0_1647948437359.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_drug_development_trials_en_3.3.4_3.0_1647948437359.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") tokenClassifier = MedicalBertForTokenClassifier.pretrained("bert_token_classifier_drug_development_trials", "en", "clinical/models")\ .setInputCols("token", "sentence")\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter]) test_sentence = """In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.""" data = spark.createDataFrame([[test_sentence]]).toDF('text') result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_drug_development_trials", "en", "clinical/models") .setInputCols(Array("token", "sentence")) .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols(Array("sentence","token","ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter)) val data = Seq("In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.drug_development_trials").predict("""In June 2003, the median overall survival with and without topotecan were 4.0 and 3.6 months, respectively. The best complete response ( CR ) , partial response ( PR ) , stable disease and progressive disease were observed in 23, 63, 55 and 33 patients, respectively, with topotecan, and 11, 61, 66 and 32 patients, respectively, without topotecan.""") ```

## Results ```bash | chunk | ner_label | |:---------------|:------------| | thyroid levels | Test | | blood test | Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_test_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_large| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 VitalTest 162 29 10 172 0.85 0.94 0.89 Test 1060 122 148 1208 0.90 0.88 0.89 Measurements 146 18 40 186 0.89 0.78 0.83 TestResult 339 80 185 524 0.81 0.65 0.72 macro_avg 1707 249 383 2090 0.86 0.81 0.83 micro_avg 1707 249 383 2090 0.87 0.82 0.84 ``` --- layout: model title: Yoruba Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_ner_yoruba date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, yo, open_source] task: Named Entity Recognition language: yo edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-ner-yoruba` is a Yoruba model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_yoruba_yo_3.4.2_3.0_1652808655534.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_yoruba_yo_3.4.2_3.0_1652808655534.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_ner_yoruba| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|yo| |Size:|773.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-ner-yoruba - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman/NERTransfer - ht --- layout: model title: Russian T5ForConditionalGeneration Cased model (from Kateryna) author: John Snow Labs name: t5_eva_forum_headlines date: 2023-01-30 tags: [ru, open_source, t5, tensorflow] task: Text Generation language: ru edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `eva_ru_forum_headlines` is a Russian model originally trained by `Kateryna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_eva_forum_headlines_ru_4.3.0_3.0_1675101795155.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_eva_forum_headlines_ru_4.3.0_3.0_1675101795155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_eva_forum_headlines","ru") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_eva_forum_headlines","ru") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_eva_forum_headlines| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ru| |Size:|981.6 MB| ## References - https://huggingface.co/Kateryna/eva_ru_forum_headlines - https://github.com/KaterynaD/eva.ru/tree/main/Code/Notebooks/9.%20Headlines --- layout: model title: ALBERT Base CoNNL-03 NER Pipeline author: John Snow Labs name: albert_base_token_classifier_conll03_pipeline date: 2022-06-19 tags: [open_source, ner, token_classifier, albert, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [albert_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/albert_base_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655654368506.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_base_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655654368506.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------+---------+ |chunk |ner_label| +--------------+---------+ |John |PER | |John Snow Labs|ORG | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_base_token_classifier_conll03_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|43.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - AlbertForTokenClassification - NerConverter - Finisher --- layout: model title: Part of Speech for Traditional Chinese author: John Snow Labs name: pos_ud_gsd_trad date: 2021-01-25 task: Part of Speech Tagging language: zh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [pos, zh, open_source] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates the part of speech of tokens in a text. The parts of speech annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 13 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. ## Predicted Entities `ADJ`, `ADP`, `ADV`, `AUX`, `CONJ`, `DET`, `NOUN`, `NUM`, `PART`, `PRON`, `PROPN`, `PUNCT`, `SYM`, `VERB`, and `X`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_trad_zh_2.7.0_2.4_1611578220288.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_gsd_trad_zh_2.7.0_2.4_1611578220288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline after tokenization.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud_trad", "zh")\ .setInputCols(["sentence"])\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_ud_gsd_trad", "zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, word_segmenter, posTagger ]) example = spark.createDataFrame([['然而，這樣的處理也衍生了一些問題。']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud_trad", "zh") .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_ud_gsd_trad", "zh") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, word_segmenter, pos)) val data = Seq("然而，這樣的處理也衍生了一些問題。").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""然而，這樣的處理也衍生了一些問題。"""] pos_df = nlu.load('zh.pos.ud_gsd_trad').predict(text, output_level = "token") pos_df ```

## Results ```bash +------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ |text |result | +------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ |然而，這樣的處理也衍生了一些問題。 |[ADV, PUNCT, PRON, PART, NOUN, ADV, VERB, PART, ADJ, NOUN, PUNCT] | +------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_gsd_trad| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|zh| ## Data Source The model was trained on the [Universal Dependencies](https://universaldependencies.org/) for Traditional Chinese annotated and converted by Google. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.70 | 0.68 | 0.69 | 272 | | ADP | 0.85 | 0.86 | 0.85 | 535 | | ADV | 0.90 | 0.90 | 0.90 | 549 | | AUX | 0.88 | 0.88 | 0.88 | 281 | | CCONJ | 0.92 | 0.87 | 0.89 | 191 | | DET | 0.93 | 0.93 | 0.93 | 138 | | NOUN | 0.88 | 0.92 | 0.90 | 3312 | | NUM | 0.98 | 0.99 | 0.98 | 653 | | PART | 0.97 | 0.94 | 0.95 | 1359 | | PRON | 0.97 | 0.97 | 0.97 | 168 | | PROPN | 0.89 | 0.84 | 0.86 | 1006 | | PUNCT | 1.00 | 1.00 | 1.00 | 1688 | | SYM | 1.00 | 1.00 | 1.00 | 3 | | VERB | 0.86 | 0.83 | 0.85 | 1769 | | X | 1.00 | 0.88 | 0.93 | 88 | | accuracy | | | 0.91 | 12012 | | macro avg | 0.91 | 0.90 | 0.91 | 12012 | | weighted avg | 0.91 | 0.91 | 0.91 | 12012 | ``` --- layout: model title: English image_classifier_vit_finetuned_chest_xray_pneumonia ViTForImageClassification from nickmuchi author: John Snow Labs name: image_classifier_vit_finetuned_chest_xray_pneumonia date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_finetuned_chest_xray_pneumonia` is a English model originally trained by nickmuchi. ## Predicted Entities `NORMAL`, `PNEUMONIA` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_finetuned_chest_xray_pneumonia_en_4.1.0_3.0_1660165538802.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_finetuned_chest_xray_pneumonia_en_4.1.0_3.0_1660165538802.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_finetuned_chest_xray_pneumonia", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_finetuned_chest_xray_pneumonia", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_finetuned_chest_xray_pneumonia| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Recognize Entities DL Pipeline for Russian - Small author: John Snow Labs name: entity_recognizer_sm date: 2021-03-22 tags: [open_source, russian, entity_recognizer_sm, pipeline, ru] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: ru edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_ru_3.0.0_3.0_1616441765899.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_ru_3.0.0_3.0_1616441765899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_sm', lang = 'ru') annotations = pipeline.fullAnnotate(""Здравствуйте из Джона Снежных Лабораторий! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_sm", lang = "ru") val result = pipeline.fullAnnotate("Здравствуйте из Джона Снежных Лабораторий! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Здравствуйте из Джона Снежных Лабораторий! ""] result_df = nlu.load('ru.ner').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:------------------------------------------------|:-----------------------------------------------|:-----------------------------------------------------------|:-----------------------------|:--------------------------------------|:-------------------------------| | 0 | ['Здравствуйте из Джона Снежных Лабораторий! '] | ['Здравствуйте из Джона Снежных Лабораторий!'] | ['Здравствуйте', 'из', 'Джона', 'Снежных', 'Лабораторий!'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['Джона Снежных Лабораторий!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|ru| --- layout: model title: Arabic Part of Speech Tagger (DA-Dialectal Arabic dataset, Egyptian Arabic POS) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_da_pos_egy date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-da-pos-egy` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_egy_ar_3.4.2_3.0_1650993413426.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_egy_ar_3.4.2_3.0_1650993413426.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_da_pos_egy| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-da-pos-egy - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: Bulgarian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_bulgarian_legal date: 2023-02-16 tags: [bul, bulgarian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: bul edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-bulgarian-roberta-base` is a Bulgarian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_bulgarian_legal_bul_4.2.4_3.0_1676558917204.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_bulgarian_legal_bul_4.2.4_3.0_1676558917204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_bulgarian_legal", "bul")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_bulgarian_legal", "bul") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_bulgarian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|bul| |Size:|416.1 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-bulgarian-roberta-base --- layout: model title: Danish BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_da_cased date: 2022-12-02 tags: [da, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: da edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-da-cased` is a Danish model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_da_cased_da_4.2.4_3.0_1670016470135.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_da_cased_da_4.2.4_3.0_1670016470135.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_da_cased","da") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_da_cased","da") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_da_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|da| |Size:|389.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-da-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English BertForQuestionAnswering Cased model (from WounKai) author: John Snow Labs name: bert_qa_wounkai_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `WounKai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_wounkai_finetuned_squad_en_4.0.0_3.0_1657186291604.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_wounkai_finetuned_squad_en_4.0.0_3.0_1657186291604.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_wounkai_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_wounkai_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_wounkai_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/WounKai/bert-finetuned-squad --- layout: model title: Pipeline to Detect PHI for Deidentification author: John Snow Labs name: ner_deidentify_dl_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, deidentification, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deidentify_dl](https://nlp.johnsnowlabs.com/2021/03/31/ner_deidentify_dl_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_pipeline_en_3.4.1_3.0_1647871253846.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deidentify_dl_pipeline_en_3.4.1_3.0_1647871253846.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deidentify_dl_pipeline", "en", "clinical/models") pipeline.annotate("A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street") ``` ```scala val pipeline = new PretrainedPipeline("ner_deidentify_dl_pipeline", "en", "clinical/models") pipeline.annotate("A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deidentify.pipeline").predict("""A . Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson , Ora MR . # 7194334 Date : 01/13/93 PCP : Oliveira , 25 month years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital . 0295 Keats Street""") ```

## Results ```bash +---------------+-----+ |ner_label |count| +---------------+-----+ |O |28 | |I-HOSPITAL |4 | |B-DATE |3 | |I-STREET |3 | |I-PATIENT |2 | |B-DOCTOR |2 | |B-AGE |1 | |B-PATIENT |1 | |I-DOCTOR |1 | |B-MEDICALRECORD|1 | +---------------+-----+. +-----------------------------+-------------+ |chunk |ner_label | +-----------------------------+-------------+ |2093-01-13 |DATE | |David Hale |DOCTOR | |Hendrickson , Ora |PATIENT | |7194334 |MEDICALRECORD| |01/13/93 |DATE | |Oliveira |DOCTOR | |25 |AGE | |2079-11-09 |DATE | |Cocke County Baptist Hospital|HOSPITAL | |0295 Keats Street |STREET | +-----------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deidentify_dl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Earning Calls Financial NER (Generic, md) author: John Snow Labs name: finner_earning_calls_generic_md date: 2022-12-15 tags: [en, finance, earning, calls, licensed, ner] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a `md` (medium) version of a financial model trained on Earning Calls transcripts to detect financial entities (NER model). This model is called `Generic` as it has fewer labels in comparison with the `Specific` version. Please note this model requires some tokenization configuration to extract the currency (see python snippet below). The currently available entities are: - AMOUNT: Numeric amounts, not percentages - ASSET: Current or Fixed Asset - ASSET_DECREASE: Decrease in the asset possession/exposure - ASSET_INCREASE: Increase in the asset possession/exposure - CF: Total cash flow - CF_DECREASE: Relative decrease in cash flow - CF_INCREASE: Relative increase in cash flow - COUNT: Number of items (not monetary, not percentages). - CURRENCY: The currency of the amount - DATE: Generic dates in context where either it's not a fiscal year or it can't be asserted as such given the context - EXPENSE: An expense or loss - EXPENSE_DECREASE: A piece of information saying there was an expense decrease in that fiscal year - EXPENSE_INCREASE: A piece of information saying there was an expense increase in that fiscal year - FCF: Free Cash Flow - FISCAL_YEAR: A date which expresses which month the fiscal exercise was closed for a specific year - KPI: Key Performance Indicator, a quantifiable measure of performance over time for a specific objective - KPI_DECREASE: Relative decrease in a KPI - KPI_INCREASE: Relative increase in a KPI - LIABILITY: Current or Long-Term Liability (not from stockholders) - LIABILITY_DECREASE: Relative decrease in liability - LIABILITY_INCREASE: Relative increase in liability - ORG: Mention to a company/organization name - PERCENTAGE: : Numeric amounts which are percentages - PROFIT: Profit or also Revenue - PROFIT_DECLINE: A piece of information saying there was a profit / revenue decrease in that fiscal year - PROFIT_INCREASE: A piece of information saying there was a profit / revenue increase in that fiscal year - TICKER: Trading symbol of the company ## Predicted Entities `AMOUNT`, `ASSET`, `ASSET_DECREASE`, `ASSET_INCREASE`, `CF`, `CF_INCREASE`, `COUNT`, `CURRENCY`, `DATE`, `EXPENSE`, `EXPENSE_DECREASE`, `EXPENSE_INCREASE`, `FCF`, `FISCAL_YEAR`, `KPI`, `KPI_DECREASE`, `KPI_INCREASE`, `LIABILITY`, `LIABILITY_DECREASE`, `LIABILITY_INCREASE`, `ORG`, `PERCENTAGE`, `PROFIT`, `PROFIT_DECLINE`, `PROFIT_INCREASE`, `TICKER`, `CF_DECREASE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_generic_md_en_1.0.0_3.0_1671135709181.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_earning_calls_generic_md_en_1.0.0_3.0_1671135709181.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token")\ .setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '”', '’', '$','€']) embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) ner_model = finance.NerModel.pretrained("finner_earning_calls_generic_md", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Adjusted EPS was ahead of our expectations at $ 1.21 , and free cash flow is also ahead of our expectations despite a $ 1.5 billion additional tax payment we made related to the R&D amortization."""]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) result.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \ .select(F.expr("cols['0']").alias("text"), F.expr("cols['1']['entity']").alias("label")).show(200, truncate = False) ```

## Results ```bash +------------+----------+----------+ | token| ner_label|confidence| +------------+----------+----------+ | Adjusted| B-PROFIT| 0.9641| | EPS| I-PROFIT| 0.9838| | was| O| 1.0| | ahead| O| 1.0| | of| O| 1.0| | our| O| 1.0| |expectations| O| 1.0| | at| O| 1.0| | $|B-CURRENCY| 1.0| | 1.21| B-AMOUNT| 1.0| | ,| O| 0.9984| | and| O| 1.0| | free| B-FCF| 0.9981| | cash| I-FCF| 0.9994| | flow| I-FCF| 0.9996| | is| O| 1.0| | also| O| 1.0| | ahead| O| 1.0| | of| O| 1.0| | our| O| 1.0| |expectations| O| 1.0| | despite| O| 1.0| | a| O| 1.0| | $|B-CURRENCY| 1.0| | 1.5| B-AMOUNT| 1.0| | billion| I-AMOUNT| 0.9999| | additional| O| 0.9786| | tax| O| 0.9603| | payment| O| 0.9043| | we| O| 1.0| | made| O| 1.0| | related| O| 1.0| | to| O| 1.0| | the| O| 0.9999| | R&D| O| 0.9993| |amortization| O| 0.9976| | .| O| 1.0| +------------+----------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_earning_calls_generic_md| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References In-house annotations on Earning Calls. ## Benchmarking ```bash label precision recall f1 support AMOUNT 99.476440 99.650350 99.563319 573 ASSET 54.838710 50.000000 52.307692 31 ASSET_INCREASE 100.000000 33.333333 50.000000 1 CF 44.827586 54.166667 49.056604 29 COUNT 61.290323 86.363636 71.698113 31 CURRENCY 99.095841 99.636364 99.365367 553 DATE 88.304094 96.178344 92.073171 171 EXPENSE 66.666667 50.602410 57.534247 63 EXPENSE_DECREASE 100.000000 60.000000 75.000000 3 EXPENSE_INCREASE 55.555556 55.555556 55.555556 9 FCF 78.947368 75.000000 76.923077 19 KPI 31.578947 28.571429 30.000000 19 KPI_DECREASE 33.333333 20.000000 25.000000 6 KPI_INCREASE 53.333333 38.095238 44.444444 15 LIABILITY 41.666667 47.619048 44.444444 24 LIABILITY_DECREASE 100.000000 33.333333 50.000000 1 ORG 95.000000 95.000000 95.000000 20 PERCENTAGE 98.951049 99.472759 99.211218 572 PROFIT 77.973568 78.666667 78.318584 227 PROFIT_DECLINE 48.648649 48.648649 48.648649 37 PROFIT_INCREASE 69.285714 66.896552 68.070175 140 TICKER 94.736842 100.000000 97.297297 19 accuracy - - 0.9585 19083 macro-avg 0.6513 0.5875 0.6067 19083 weighted-avg 0.9577 0.9585 0.9577 19083 ``` --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_final_784824211 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-final-784824211` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824211_en_4.3.1_3.0_1678783263483.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_final_784824211_en_4.3.1_3.0_1678783263483.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824211","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_final_784824211","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_hadxu_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_hadxu_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_hadxu_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/hadxu/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from annt5396) author: John Snow Labs name: distilbert_qa_annt5396_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `annt5396`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_annt5396_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769820445.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_annt5396_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769820445.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_annt5396_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_annt5396_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_annt5396_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/annt5396/distilbert-base-uncased-finetuned-squad --- layout: model title: German asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922_de_4.2.0_3.0_1664119967685.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922_de_4.2.0_3.0_1664119967685.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_gender_male_0_female_10_s922| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English asr_xlsr_wav2vec2_final TFWav2Vec2ForCTC from chrisvinsen author: John Snow Labs name: asr_xlsr_wav2vec2_final date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xlsr_wav2vec2_final` is a English model originally trained by chrisvinsen. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xlsr_wav2vec2_final_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_final_en_4.2.0_3.0_1664110483097.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xlsr_wav2vec2_final_en_4.2.0_3.0_1664110483097.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xlsr_wav2vec2_final", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xlsr_wav2vec2_final", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xlsr_wav2vec2_final| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering model (from tiennvcs) author: John Snow Labs name: bert_qa_bert_large_uncased_finetuned_docvqa date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-finetuned-docvqa` is a English model orginally trained by `tiennvcs`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_finetuned_docvqa_en_4.0.0_3.0_1654536476096.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_finetuned_docvqa_en_4.0.0_3.0_1654536476096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_finetuned_docvqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_finetuned_docvqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.large_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_finetuned_docvqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tiennvcs/bert-large-uncased-finetuned-docvqa --- layout: model title: Language Detection & Identification Pipeline - 231 Languages author: John Snow Labs name: detect_language_231 date: 2020-12-05 task: [Pipeline Public, Language Detection, Sentence Detection] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, pipeline, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNN architectures in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This pipeline can detect the following languages: ## Predicted Entities `Achinese`, `Afrikaans`, `Tosk Albanian`, `Amharic`, `Aragonese`, `Old English`, `Arabic`, `Egyptian Arabic`, `Assamese`, `Asturian`, `Avaric`, `Aymara`, `Azerbaijani`, `South Azerbaijani`, `Bashkir`, `Bavarian`, `bat-smg`, `Central Bikol`, `Belarusian`, `Bulgarian`, `bh`, `Banjar`, `Bengali`, `Tibetan`, `Bishnupriya`, `Breton`, `Bosnian`, `Russia Buriat`, `Catalan`, `cbk-zam`, `Min Dong Chinese`, `Chechen`, `Cebuano`, `Central Kurdish (Soranî)`, `Corsican`, `Crimean Tatar`, `Czech`, `Kashubian`, `Chuvash`, `Welsh`, `Danish`, `German`, `Dimli (individual language)`, `Lower Sorbian`, `dty`, `Dhivehi`, `Greek`, `eml`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Extremaduran`, `Persian`, `Finnish`, `fiu-vro`, `Faroese`, `French`, `Arpitan`, `Friulian`, `Frisian`, `Irish`, `Gagauz`, `Scottish Gaelic`, `Galician`, `Gilaki`, `Guarani`, `Konkani (Goan)`, `Gujarati`, `Manx`, `Hausa`, `Hakka Chinese`, `Hebrew`, `Hindi`, `Fiji Hindi`, `Croatian`, `Upper Sorbian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Interlingue`, `Igbo`, `Ilocano`, `Ido`, `Icelandic`, `Italian`, `Japanese`, `Jamaican Patois`, `Lojban`, `Javanese`, `Georgian`, `Karakalpak`, `Kabyle`, `Kabardian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Komi-Permyak`, `Karachay-Balkar`, `Kölsch`, `Kurdish`, `Komi`, `Cornish`, `Kyrgyz`, `Latin`, `Ladino`, `Luxembourgish`, `Lezghian`, `Luganda`, `Limburgan`, `Ligurian`, `Lombard`, `Lingala`, `Lao`, `Northern Luri`, `Lithuanian`, `Latgalian`, `Latvian`, `Maithili`, `map-bms`, `Moksha`, `Malagasy`, `Meadow Mari`, `Maori`, `Minangkabau`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Hill Mari`, `Malay`, `Maltese`, `Mirandese`, `Burmese`, `Erzya`, `Mazanderani`, `Nahuatl`, `Neapolitan`, `Low German (Low Saxon)`, `nds-nl`, `Nepali`, `Newari`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Narom`, `Pedi`, `Navajo`, `Occitan`, `Livvi`, `Oromo`, `Odia (Oriya)`, `Ossetian`, `Punjabi (Eastern)`, `Pangasinan`, `Kapampangan`, `Papiamento`, `Picard`, `Pennsylvania German`, `Palatine German`, `Polish`, `Punjabi (Western)`, `Pashto`, `Portuguese`, `Quechua`, `Romansh`, `Romanian`, `roa-rup`, `roa-tara`, `Russian`, `Rusyn`, `Kinyarwanda`, `Sanskrit`, `Yakut`, `Sardinian`, `Sicilian`, `Sindhi`, `Northern Sami`, `Serbo-Croatian`, `Sinhala`, `Slovak`, `Slovenian`, `Shona`, `Somali`, `Albanian`, `Serbian`, `Sranan Tongo`, `Saterland Frisian`, `Sundanese`, `Swedish`, `Swahili`, `Silesian`, `Tamil`, `Tulu`, `Telugu`, `Tetun`, `Tajik`, `Thai`, `Turkmen`, `Tagalog`, `Setswana`, `Tongan`, `Turkish`, `Tatar`, `Tuvinian`, `Udmurt`, `Uyghur`, `Ukrainian`, `Urdu`, `Uzbek`, `Venetian`, `Veps`, `Vietnamese`, `Vlaams`, `Volapük`, `Walloon`, `Waray`, `Wolof`, `Shanghainese`, `Xhosa`, `Mingrelian`, `Yiddish`, `Yoruba`, `Zeeuws`, `Chinese`, `zh-classical`, `zh-min-nan`, `zh-yue`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/LANGUAGE_DETECTOR/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/detect_language_231_xx_2.7.0_2.4_1607185843755.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/detect_language_231_xx_2.7.0_2.4_1607185843755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("detect_language_231", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("detect_language_231", lang = "xx") pipeline.annotate("French author who helped pioneer the science-fiction genre.") ``` {:.nlu-block} ```python import nlu text = ["French author who helped pioneer the science-fiction genre."] lang_df = nlu.load("xx.classify.lang.231").predict(text) lang_df ```

## Results ```bash {'document': ['French author who helped pioneer the science-fiction genre.'], 'language': ['en']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|detect_language_231| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - LanguageDetectorDL --- layout: model title: Detect Entities in Urdu (urduvec_140M_300d embeddings) author: John Snow Labs name: uner_mk_140M_300d date: 2022-08-09 tags: [ner, ur, open_source] task: Named Entity Recognition language: ur edition: Spark NLP 4.0.2 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses Urdu word embeddings to find 7 different types of entities in Urdu text. It is trained using `urduvec_140M_300d` word embeddings, so please use the same embeddings in the pipeline. Predicted Entities : Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Dates-``DATE``, Designations-``DESIGNATION``, Times-``TIME``, Numbers-``NUMBER``. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE`, `TIME`, `DESIGNATION`, `NUMBER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/uner_mk_140M_300d_ur_4.0.2_3.0_1660035998466.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/uner_mk_140M_300d_ur_4.0.2_3.0_1660035998466.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("uner_mk_140M_300d", "ur") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("""بریگیڈیئر ایڈ بٹلر سنہ دوہزارچھ میں ہلمند کے فوجی کمانڈر تھے۔""") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("uner_mk_140M_300d", "ur") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val nlp_pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, ner, ner_converter)) val data = Seq("""بریگیڈیئر ایڈ بٹلر سنہ دوہزارچھ میں ہلمند کے فوجی کمانڈر تھے۔""").toDS.toDF("text") val result = nlp_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ur.ner").predict("""بریگیڈیئر ایڈ بٹلر سنہ دوہزارچھ میں ہلمند کے فوجی کمانڈر تھے۔""") ```

## Results ```bash | | ner_chunk | entity | |---:|---------------:|-------------:| | 0 |بریگیڈیئر | DESIGNATION | | 1 |ایڈ بٹلر | PERSON | | 2 |سنہ دوہزارچھ | DATE | | 3 |ہلمند | LOCATION | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|uner_mk_140M_300d| |Type:|ner| |Compatibility:|Spark NLP 4.0.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|ur| |Size:|14.8 MB| ## References This model is trained using the following datasets: https://www.researchgate.net/publication/312218764_Named_Entity_Dataset_for_Urdu_Named_Entity_Recognition_Task https://www.researchgate.net/publication/332653135_Urdu_Named_Entity_Recognition_Corpus_Generation_and_Deep_Learning_Applications ## Benchmarking ```bash label tp fp fn prec rec f1 I-TIME 12 10 1 0.545455 0.923077 0.685714 B-PERSON 2808 846 535 0.768473 0.839964 0.80263 B-DATE 34 6 6 0.85 0.85 0.85 I-DATE 45 1 2 0.978261 0.957447 0.967742 B-DESIGNATION 49 30 16 0.620253 0.753846 0.680556 I-LOCATION 2110 750 701 0.737762 0.750623 0.744137 B-TIME 11 9 3 0.55 0.785714 0.647059 I-ORGANIZATION 2006 772 760 0.722102 0.725235 0.723665 I-NUMBER 18 6 2 0.75 0.9 0.818182 B-LOCATION 5428 1255 582 0.81221 0.903161 0.855275 B-NUMBER 194 36 27 0.843478 0.877828 0.86031 I-DESIGNATION 25 15 6 0.625 0.806452 0.704225 I-PERSON 3562 759 433 0.824346 0.891614 0.856662 B-ORGANIZATION 1114 466 641 0.705063 0.634758 0.668066 Macro-average 17416 4961 3715 0.738029 0.828551 0.780675 Micro-average 17416 4961 3715 0.778299 0.824192 0.800588 ``` --- layout: model title: Chinese Financial NER (sm, bert_embeddings_mengzi_bert_base_fin) author: John Snow Labs name: finner_finance_chinese_sm date: 2023-02-04 tags: [zh, cn, finance, ner, licensed] task: Named Entity Recognition language: zh edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: FinanceNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is the small version of the NER model for Financial Chinese texts, trained in a subset of **ChFinAnn** (see "Datasets used for training"). To use this model, use the `BertEmbeddings` model named `bert_embeddings_mengzi_bert_base_fin"` as: ```python bert_embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_mengzi_bert_base_fin","zh") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ``` Also, please note that the Chinese texts are not separated by white space. The embedding model we use is based on character-level embeddings, so you need to split the text on every character (for example, by setting `.setSplitPattern("")` in the `Tokenizer` annotator). ## Predicted Entities `AveragePrice`, `ClosingDate`, `CompanyName`, `EndDate`, `EquityHolder`, `FrozeShares`, `HighestTradingPrice`, `LaterHoldingShares`, `LegalInstitution`, `LowestTradingPrice`, `OtherType`, `PledgedShares`, `Pledgee`, `ReleasedDate`, `RepurchaseAmount`, `RepurchasedShares`, `StartDate`, `StockAbbr`, `StockCode`, `TotalHoldingRatio`, `TotalHoldingShares`, `TotalPledgedShares`, `TradedShares`, `UnfrozeDate` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_FINANCE_CHINESE){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_finance_chinese_sm_zh_1.0.0_3.0_1675554138686.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_finance_chinese_sm_zh_1.0.0_3.0_1675554138686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = nlp.Tokenizer()\ .setInputCols(["document"])\ .setOutputCol("token")\ .setSplitPattern("") # Split on char level embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_mengzi_bert_base_fin","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained("finner_finance_chinese_sm", "zh", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""近日，渤海水业股份有限公司（以下简称“公司”）收到公司持股5%以上股东李华青女士的《告知函》，获悉李华青女士将其所持有的部分公司股票进行补充质押，具体事项如下："""] res = model. Transform(spark.createDataFrame([text]).toDF("text")) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['0']").alias("chunk"), F.expr("cols['1']['entity']").alias("ner_label"), F.expr("cols['1']['confidence']").alias("confidence")).show(truncate=False) ```

## Results ```bash +------------------------------------+------------+----------+ |chunk |ner_label |confidence| +------------------------------------+------------+----------+ |业股份有限公司（以下简称“公司”）收到|CompanyName |0.7933 | |质押，具体 |EquityHolder|0.9378667 | +------------------------------------+------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_finance_chinese_sm| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|zh| |Size:|16.8 MB| ## References The dataset used for training was a subset of the **chFinAnn** dataset, consisting of financial statements of Chinese listed companies from 2008 to 2018. Reference: - [Doc2EDAG: An End-to-End Document-level Framework for Chinese Financial Event Extraction](https://aclanthology.org/D19-1032) (Zheng et al., EMNLP-IJCNLP 2019) ## Sample text from the training dataset 近日，渤海水业股份有限公司（以下简称“公司”）收到公司持股5%以上股东李华青女士的《告知函》，获悉李华青女士将其所持有的部分公司股票进行补充质押，具体事项如下： ## Benchmarking ```bash entity precision recall f1 support AveragePrice 78.0731 85.1449 81.4558 301 ClosingDate 76.0148 57.7031 65.6051 271 CompanyName 94.0767 95.9251 94.9919 5605 EndDate 75.487 44.5402 56.0241 616 EquityHolder 83.8303 91.319 87.4146 7780 FrozeShares 47.4227 31.7241 38.0165 97 HighestTradingPrice 74.4186 70.5085 72.4108 559 LaterHoldingShares 31.4961 11.9048 17.2786 127 LegalInstitution 92.3767 87.6596 89.9563 223 LowestTradingPrice 78.5047 52.1739 62.6866 107 OtherType 78.2961 36.7619 50.0324 493 PledgedShares 78.0776 65.5189 71.249 1779 Pledgee 90.1003 88.656 89.3723 1596 ReleasedDate 54.5016 46.1853 50 622 RepurchaseAmount 68.323 72.8477 70.5128 322 RepurchasedShares 79.5918 70.4819 74.7604 588 StartDate 65.4217 77.5493 70.9711 4150 StockAbbr 83.7656 82.0521 82.9 2969 StockCode 99.8355 99.5626 99.6989 1824 TotalHoldingRatio 74.2574 85.1628 79.3371 1515 TotalHoldingShares 67.0582 88.7306 76.387 2043 TotalPledgedShares 74.5989 86.0226 79.9045 1122 TradedShares 76.9231 68.6948 72.5765 910 UnfrozeDate 5.88235 5.55556 5.71429 17 ``` --- layout: model title: Pipeline to Resolve ICD-9-CM Codes author: John Snow Labs name: icd9_resolver_pipeline date: 2023-03-29 tags: [en, licensed, clinical, resolver, pipeline, chunk_mapping, icd9] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities with their corresponding ICD-9-CM codes. You’ll just feed your text and it will return the corresponding ICD-9-CM codes. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd9_resolver_pipeline_en_4.3.2_3.2_1680119796842.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd9_resolver_pipeline_en_4.3.2_3.2_1680119796842.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd9_resolver_pipeline", "en", "clinical/models") text= A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years and anisakiasis. Also, it was reported that fetal and neonatal hemorrhage result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd9_resolver_pipeline", "en", "clinical/models") val result = pipeline.fullAnnotate(A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years and anisakiasis. Also, it was reported that fetal and neonatal hemorrhage) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd9.pipeline").predict("""Put your text here.""") ```

## Results ```bash +-----------------------------+---------+---------+ |chunk |ner_chunk|icd9_code| +-----------------------------+---------+---------+ |gestational diabetes mellitus|PROBLEM |V12.21 | |anisakiasis |PROBLEM |127.1 | |fetal and neonatal hemorrhage|PROBLEM |772 | +-----------------------------+---------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd9_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|2.2 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: Voice of the Patients (embeddings_clinical_large) author: John Snow Labs name: ner_vop_emb_clinical_large_wip date: 2023-04-12 tags: [licensed, clinical, en, ner, vop, patient] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.0 spark_version: [3.2, 3.0] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Allergen`, `SubstanceQuantity`, `RaceEthnicity`, `Measurements`, `InjuryOrPoisoning`, `Treatment`, `TestResult`, `Modifier`, `Route`, `MedicalDevice`, `Vaccine`, `RelationshipStatus`, `Frequency`, `HealthStatus`, `Procedure`, `Duration`, `DateTime`, `Disease`, `Test`, `Substance`, `Symptom`, `Laterality`, `Dosage`, `ClinicalDept`, `PsychologicalCondition`, `VitalTest`, `Age`, `Drug`, `BodyPart`, `AdmissionDischarge`, `Form`, `Employment`, `Gender` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/VOICE_OF_THE_PATIENTS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_large_wip_en_4.4.0_3.0_1681315187438.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_large_wip_en_4.4.0_3.0_1681315187438.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_false_positives_scancode_base_uncased_l8_1","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_false_positives_scancode_base_uncased_l8_1","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_false_positives_scancode_base_uncased_l8_1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/ayansinha/false-positives-scancode-bert-base-uncased-L8-1 - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer - https://github.com/nexB/scancode-results-analyzer#quickstart---local-machine - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py - https://github.com/nexB/scancode-results-analyzer/blob/master/src/results_analyze/nlp_models.py --- layout: model title: English RobertaForQuestionAnswering (from eAsyle) author: John Snow Labs name: roberta_qa_testABSA3 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `testABSA3` is a English model originally trained by `eAsyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_testABSA3_en_4.0.0_3.0_1655739938744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_testABSA3_en_4.0.0_3.0_1655739938744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_testABSA3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_testABSA3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.testabsa3.by_eAsyle").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_testABSA3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|426.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/eAsyle/testABSA3 --- layout: model title: Diseases and Syndromes to UMLS Code Pipeline author: John Snow Labs name: umls_disease_syndrome_resolver_pipeline date: 2023-03-30 tags: [en, licensed, umls, pipeline] task: Pipeline Healthcare language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities (Diseases and Syndromes) with their corresponding UMLS CUI codes. You’ll just feed your text and it will return the corresponding UMLS codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_disease_syndrome_resolver_pipeline_en_4.3.2_3.2_1680190919515.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_disease_syndrome_resolver_pipeline_en_4.3.2_3.2_1680190919515.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------+---------+---------+ |chunk |ner_label|umls_code| +-----------------------------+---------+---------+ |poor appetite |PROBLEM |C0003123 | |gestational diabetes mellitus|PROBLEM |C0085207 | |acyclovir allergy |PROBLEM |C0571297 | |polyuria |PROBLEM |C0018965 | +-----------------------------+---------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_disease_syndrome_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.4 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: English BertForQuestionAnswering Uncased model (from husnu) author: John Snow Labs name: bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xtremedistil-l6-h256-uncased-TQUAD-finetuned_lr-2e-05_epochs-3` is a English model originally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3_en_4.0.0_3.0_1657193231850.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3_en_4.0.0_3.0_1657193231850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_xtremedistil_l6_h256_uncased_tquad_finetuned_lr_2e_05_epochs_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|47.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/husnu/xtremedistil-l6-h256-uncased-TQUAD-finetuned_lr-2e-05_epochs-3 --- layout: model title: Italian Bert Embeddings (from Geotrend) author: John Snow Labs name: bert_embeddings_bert_base_it_cased date: 2022-04-11 tags: [bert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-it-cased` is a Italian model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_it_cased_it_3.4.2_3.0_1649676920355.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_it_cased_it_3.4.2_3.0_1649676920355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_it_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|396.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-it-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Legal Standard terms and conditions of trust Clause Binary Classifier author: John Snow Labs name: legclf_standard_terms_and_conditions_of_trust_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `standard-terms-and-conditions-of-trust` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `standard-terms-and-conditions-of-trust` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_standard_terms_and_conditions_of_trust_clause_en_1.0.0_3.2_1660123035837.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_standard_terms_and_conditions_of_trust_clause_en_1.0.0_3.2_1660123035837.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_standard_terms_and_conditions_of_trust_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[standard-terms-and-conditions-of-trust]| |[other]| |[other]| |[standard-terms-and-conditions-of-trust]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_standard_terms_and_conditions_of_trust_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.75 1.00 0.86 6 standard-terms-and-conditions-of-trust 0.00 0.00 0.00 2 accuracy - - 0.75 8 macro-avg 0.38 0.50 0.43 8 weighted-avg 0.56 0.75 0.64 8 ``` --- layout: model title: English image_classifier_vit_CarViT ViTForImageClassification from abdusahmbzuai author: John Snow Labs name: image_classifier_vit_CarViT date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_CarViT` is a English model originally trained by abdusahmbzuai. ## Predicted Entities `Toyota`, `Audi`, `Dodge`, `Aston Martin`, `Chevrolet`, `Mitsubishi`, `Kia`, `Honda`, `Chrysler`, `Lexus`, `Land Rover`, `Rolls-Royce`, `Porsche`, `FIAT`, `Cadillac`, `Jaguar`, `smart`, `Tesla`, `Maserati`, `Buick`, `GMC`, `Genesis`, `McLaren`, `Bentley`, `BMW`, `Lincoln`, `Subaru`, `Volvo`, `Lamborghini`, `Nissan`, `Alfa Romeo`, `Jeep`, `INFINITI`, `Mazda`, `Hyundai`, `Volkswagen`, `Ram`, `Ferrari`, `Acura`, `Mercedes-Benz`, `MINI`, `Ford` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_CarViT_en_4.1.0_3.0_1660165745338.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_CarViT_en_4.1.0_3.0_1660165745338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_CarViT", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_CarViT", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_CarViT| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.0 MB| --- layout: model title: English BertForQuestionAnswering model (from SreyanG-NVIDIA) author: John Snow Labs name: bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad` is a English model orginally trained by `SreyanG-NVIDIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181017940.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181017940.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_SreyanG_NVIDIA_bert_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SreyanG-NVIDIA/bert-base-uncased-finetuned-squad --- layout: model title: Kanuri asr_wav2vec2_xlsr_korean_senior TFWav2Vec2ForCTC from hyyoka author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_korean_senior date: 2022-09-24 tags: [wav2vec2, kr, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: kr edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_korean_senior` is a Kanuri model originally trained by hyyoka. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_korean_senior_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_korean_senior_kr_4.2.0_3.0_1664024430198.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_korean_senior_kr_4.2.0_3.0_1664024430198.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_korean_senior', lang = 'kr') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_korean_senior", lang = "kr") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_korean_senior| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|kr| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Base Cased model (from razent) author: John Snow Labs name: t5_scifive_base_pubmed date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `SciFive-base-Pubmed` is a English model originally trained by `razent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_scifive_base_pubmed_en_4.3.0_3.0_1675098922792.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_scifive_base_pubmed_en_4.3.0_3.0_1675098922792.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_scifive_base_pubmed","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_scifive_base_pubmed","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_scifive_base_pubmed| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|474.3 MB| ## References - https://huggingface.co/razent/SciFive-base-Pubmed - https://arxiv.org/abs/2106.03598 - https://github.com/justinphan3110/SciFive --- layout: model title: Word2Vec Embeddings in Nahuatl languages (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, nah, open_source] task: Embeddings language: nah edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_nah_3.4.1_3.0_1647447890372.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_nah_3.4.1_3.0_1647447890372.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_roberta", "es", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = RoBertaEmbeddings.pretrained("roberta_base_biomedical", "es") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_roberta", "es", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.living_species.roberta").predict("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_pos_RuPERTa_base_finetuned_pos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|470.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-pos - https://www.kaggle.com/nltkdata/conll-corpora - https://www.kaggle.com/nltkdata/conll-corpora - https://twitter.com/mrm8488 - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: English BertForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-finetuned-chaii` is a English model orginally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii_en_4.0.0_3.0_1654536941289.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii_en_4.0.0_3.0_1654536941289.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.bert.large_uncased_uncased_whole_word_masking_finetuned.by_SauravMaheshkar").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_finetuned_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/bert-large-uncased-whole-word-masking-finetuned-chaii --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8_en_4.0.0_3.0_1657192224643.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8_en_4.0.0_3.0_1657192224643.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|375.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-8 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from marioarteaga) author: John Snow Labs name: distilbert_qa_marioarteaga_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `marioarteaga`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_marioarteaga_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772174929.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_marioarteaga_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772174929.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_marioarteaga_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_marioarteaga_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_marioarteaga_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/marioarteaga/distilbert-base-uncased-finetuned-squad --- layout: model title: Extract Community Condition Entities from Social Determinants of Health Texts author: John Snow Labs name: ner_sdoh_community_condition_wip date: 2023-02-24 tags: [licensed, en, clinical, sdoh, social_determinants, ner, public_health, community, condition, community_condition] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts community condition information related to Social Determinants of Health from various kinds of biomedical documents. ## Predicted Entities `Transportation`, `Community_Living_Conditions`, `Housing`, `Food_Insecurity` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_community_condition_wip_en_4.3.1_3.0_1677201525944.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_community_condition_wip_en_4.3.1_3.0_1677201525944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_sdoh_community_condition_wip", "en", "clinical/models")\ .setInputCols(["sentence", "token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_texts = ["He is currently experiencing financial stress due to job insecurity, and he lives in a small apartment in a densely populated area with limited access to green spaces and outdoor recreational activities.", "Patient reports difficulty affording healthy food, and relies oncheaper, processed options.", "She reports her husband and sons provide transportation top medical apptsand do her grocery shopping."] data = spark.createDataFrame(sample_texts, StringType()).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_sdoh_community_condition_wip", "en", "clinical/models") .setInputCols(Array("sentence", "token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter )) val data = Seq("He is currently experiencing financial stress due to job insecurity, and he lives in a small apartment in a densely populated area with limited access to green spaces and outdoor recreational activities.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-------------------------------+-----+---+---------------------------+ |chunk |begin|end|ner_label | +-------------------------------+-----+---+---------------------------+ |small apartment |87 |101|Housing | |green spaces |154 |165|Community_Living_Conditions| |outdoor recreational activities|171 |201|Community_Living_Conditions| |healthy food |37 |48 |Food_Insecurity | |transportation |41 |54 |Transportation | +-------------------------------+-----+---+---------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_community_condition_wip| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.0 MB| ## Benchmarking ```bash label tp fp fn total precision recall f1 Food_Insecurity 40.0 0.0 5.0 45.0 1.000000 0.888889 0.941176 Housing 376.0 20.0 28.0 404.0 0.949495 0.930693 0.940000 Community_Living_Conditions 97.0 8.0 8.0 105.0 0.923810 0.923810 0.923810 Transportation 31.0 2.0 0.0 31.0 0.939394 1.000000 0.968750 ``` --- layout: model title: Sentence Entity Resolver for Hierarchical Condition Categories (HCC) codes (Augmented) author: John Snow Labs name: sbiobertresolve_hcc_augmented date: 2023-05-31 tags: [en, entity_resolution, clinical, hcc, licensed] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to Hierarchical Condition Categories (HCC) codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. ## Predicted Entities `HCC Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_hcc_augmented_en_4.4.2_3.0_1685528056939.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_hcc_augmented_en_4.4.2_3.0_1685528056939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+-------+----------+--------------------+---------------------------------------------------------------------------+ | ner_chunk| entity|icd10_code| all_codes| resolutions| +---------------------------+-------+----------+--------------------+---------------------------------------------------------------------------+ | hypertension|PROBLEM| 0| [0]| [hypertension [essential (primary) hypertension]]| |chronic renal insufficiency|PROBLEM| 0| [0, 136]|[chronic renal insufficiency [chronic kidney disease, unspecified], end ...| | COPD|PROBLEM| 111|[111, 80, 0, 23, 47]|[copd [chronic obstructive pulmonary disease, unspecified], coning [comp...| | gastritis|PROBLEM| 0| [0]| [gastritis [gastritis, unspecified, without bleeding]]| | TIA|PROBLEM| 0| [0, 12, 48]|[tia [transient cerebral ischemic attack, unspecified], tsh-oma [benign ...| | hypotension|PROBLEM| 0| [0]| [hypotension [hypotension]]| +---------------------------+-------+----------+--------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_hcc_augmented| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[hcc]| |Language:|en| |Size:|1.4 GB| |Case sensitive:|false| --- layout: model title: English DistilBertForQuestionAnswering model (from alinemati) author: John Snow Labs name: distilbert_qa_BERT date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BERT` is a English model originally trained by `alinemati`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_BERT_en_4.0.0_3.0_1654722812832.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_BERT_en_4.0.0_3.0_1654722812832.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_BERT","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_BERT","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.by_alinemati").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_BERT| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/alinemati/BERT --- layout: model title: NER Model for Hindi+English author: John Snow Labs name: bert_token_classifier_hi_en_ner date: 2021-12-27 tags: [ner, hi, en, open_source] task: Named Entity Recognition language: hi edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from Hugging Face to carry out Name Entity Recognition with mixed Hindi-English texts, provided by the LinCE repository. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_HINDI_ENGLISH.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_hi_en_ner_hi_3.2.0_3.0_1640612846736.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_hi_en_ner_hi_3.2.0_3.0_1640612846736.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentence_detector = SentenceDetector() \ .setInputCols(['document'])\ .setOutputCol('sentence') tokenizer = Tokenizer()\ .setInputCols(['sentence']) \ .setOutputCol('token') tokenClassifier_loaded = BertForTokenClassification.pretrained("bert_token_classifier_hi_en_ner","hi")\ .setInputCols(["sentence",'token'])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, tokenClassifier_loaded, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([["""रिलायंस इंडस्ट्रीज़ लिमिटेड (Reliance Industries Limited) एक भारतीय संगुटिका नियंत्रक कंपनी है, जिसका मुख्यालय मुंबई, महाराष्ट्र (Maharashtra) में स्थित है।रतन नवल टाटा (28 दिसंबर 1937, को मुम्बई (Mumbai), में जन्मे) टाटा समुह के वर्तमान अध्यक्ष, जो भारत की सबसे बड़ी व्यापारिक समूह है, जिसकी स्थापना जमशेदजी टाटा ने की और उनके परिवार की पीढियों ने इसका विस्तार किया और इसे दृढ़ बनाया।"""]], ["text"])) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val tokenClassifier_loaded = BertForTokenClassification.pretrained("bert_token_classifier_hi_en_ner","hi") .setInputCols("sentence","token") .setOutputCol("ner") val ner_converter = NerConverter() .setInputCols("sentence","token","ner") .setOutputCol("ner_chunk") val nlp_pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, tokenClassifier_loaded, ner_converter)) val data = Seq("""रिलायंस इंडस्ट्रीज़ लिमिटेड (Reliance Industries Limited) एक भारतीय संगुटिका नियंत्रक कंपनी है, जिसका मुख्यालय मुंबई, महाराष्ट्र (Maharashtra) में स्थित है।रतन नवल टाटा (28 दिसंबर 1937, को मुम्बई (Mumbai), में जन्मे) टाटा समुह के वर्तमान अध्यक्ष, जो भारत की सबसे बड़ी व्यापारिक समूह है, जिसकी स्थापना जमशेदजी टाटा ने की और उनके परिवार की पीढियों ने इसका विस्तार किया और इसे दृढ़ बनाया।""").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("hi.ner.bert").predict("""रिलायंस इंडस्ट्रीज़ लिमिटेड (Reliance Industries Limited) एक भारतीय संगुटिका नियंत्रक कंपनी है, जिसका मुख्यालय मुंबई, महाराष्ट्र (Maharashtra) में स्थित है।रतन नवल टाटा (28 दिसंबर 1937, को मुम्बई (Mumbai), में जन्मे) टाटा समुह के वर्तमान अध्यक्ष, जो भारत की सबसे बड़ी व्यापारिक समूह है, जिसकी स्थापना जमशेदजी टाटा ने की और उनके परिवार की पीढियों ने इसका विस्तार किया और इसे दृढ़ बनाया।""") ```

## Results ```bash | chunk | ner_label | |----------------------- |-------------- | | रिलायंस इंडस्ट्रीज़ लिमिटेड | ORGANISATION | | Reliance Industries Limited | ORGANISATION | | मुंबई | PLACE | | महाराष्ट्र | PLACE | | Maharashtra | PLACE | | नवल टाटा | PERSON | | मुम्बई | PLACE | | Mumbai | PLACE | | टाटा समुह | ORGANISATION | | भारत | PLACE | | जमशेदजी टाटा | PERSON | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_hi_en_ner| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|hi| |Size:|665.7 MB| |Case sensitive:|true| |Max sentense length:|128| ## Data Source https://ritual.uh.edu/lince/home --- layout: model title: English DistilBertForTokenClassification Cased model (from ankleBowl) author: John Snow Labs name: distilbert_token_classifier_autotrain_lucy_light_control_3122788375 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-lucy-light-control-3122788375` is a English model originally trained by `ankleBowl`. ## Predicted Entities `PER`, `OFF`, `BRI`, `EMP`, `ONN`, `DIM`, `COL` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_lucy_light_control_3122788375_en_4.3.1_3.0_1678783543204.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_lucy_light_control_3122788375_en_4.3.1_3.0_1678783543204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_lucy_light_control_3122788375","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_lucy_light_control_3122788375","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_lucy_light_control_3122788375| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ankleBowl/autotrain-lucy-light-control-3122788375 --- layout: model title: English BertForQuestionAnswering Uncased model (from aodiniz) author: John Snow Labs name: bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-6_H-128_A-2_cord19-200616_squad2` is a English model originally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2_en_4.0.0_3.0_1657188925469.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2_en_4.0.0_3.0_1657188925469.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_uncased_l_6_h_128_a_2_cord19_200616_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|20.0 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-6_H-128_A-2_cord19-200616_squad2 --- layout: model title: Swiss Legal Roberta Embeddings author: John Snow Labs name: roberta_large_swiss_legal date: 2023-02-16 tags: [gsw, swiss, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: gsw edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-swiss-roberta-large` is a Swiss model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_large_swiss_legal_gsw_4.2.4_3.0_1676577298804.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_large_swiss_legal_gsw_4.2.4_3.0_1676577298804.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_large_swiss_legal", "gsw")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_large_swiss_legal", "gsw") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_large_swiss_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|gsw| |Size:|1.6 GB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-swiss-roberta-large --- layout: model title: German asr_wav2vec2_large_xlsr_53_german_cv9 TFWav2Vec2ForCTC from oliverguhr author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_german_cv9 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_german_cv9` is a German model originally trained by oliverguhr. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_german_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_cv9_de_4.2.0_3.0_1664096909834.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_german_cv9_de_4.2.0_3.0_1664096909834.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_german_cv9", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_german_cv9", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_german_cv9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Yoruba Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:37:00 +0800 task: Lemmatization language: yo edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, yo] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_yo_2.5.5_2.4_1596055008864.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_yo_2.5.5_2.4_1596055008864.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "yo") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "yo") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Yato si jijẹ ọba ariwa, John Snow jẹ oṣoogun ara Gẹẹsi kan ati adari ninu idagbasoke anaesthesia ati imototo ilera."""] lemma_df = nlu.load('yo.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=3, result='Yato', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=5, end=6, result='si', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=8, end=11, result='jijẹ', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=13, end=15, result='ọba', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=17, end=21, result='ariwa', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|yo| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English RoBERTa Embeddings (Mixed sampling strategy) author: John Snow Labs name: roberta_embeddings_distilroberta_base_climate_d_s date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilroberta-base-climate-d-s` is a English model orginally trained by `climatebert`. Sampling strategy ds:As expressed in the author's paper [here](https://arxiv.org/pdf/2110.12010.pdf), ds is "div select + sim select", meaning 70% of the biggest composite scaled score diverse+sim was used, discarding the rest. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_d_s_en_3.4.2_3.0_1649946889886.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_distilroberta_base_climate_d_s_en_3.4.2_3.0_1649946889886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_distilroberta_base_climate_d_s| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|310.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/climatebert/distilroberta-base-climate-d-s - https://arxiv.org/abs/2110.12010 --- layout: model title: Fast Neural Machine Translation Model from English to Brazilian Sign Language author: John Snow Labs name: opus_mt_en_bzs date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bzs, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bzs` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bzs_xx_2.7.0_2.4_1609168236857.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bzs_xx_2.7.0_2.4_1609168236857.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bzs| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Resolve Medication Codes(Transform) author: John Snow Labs name: medication_resolver_transform_pipeline date: 2022-09-01 tags: [resolver, rxnorm, ndc, snomed, umls, ade, pipeline, en, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained resolver pipeline to extract medications and resolve their adverse reactions (ADE), RxNorm, UMLS, NDC, SNOMED CT codes, and action/treatments in clinical text. Action/treatments are available for branded medication, and SNOMED codes are available for non-branded medication. This pipeline can be used with Spark transform. You can use `medication_resolver_pipeline` as Lightpipeline (with `annotate/fullAnnotate`). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/medication_resolver_transform_pipeline_en_4.0.2_3.0_1662045429139.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/medication_resolver_transform_pipeline_en_4.0.2_3.0_1662045429139.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline medication_resolver_pipeline = PretrainedPipeline("medication_resolver_transform_pipeline", "en", "clinical/models") text = """The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""" data = spark.createDataFrame([[text]]).toDF("text") result = medication_resolver_pipeline.transform(data) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val medication_resolver_pipeline = new PretrainedPipeline("medication_resolver_transform_pipeline", "en", "clinical/models") val data = Seq("""The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""").toDS.toDF("text") val result = medication_resolver_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.medication_transform.pipeline").predict("""The patient was prescribed Amlodopine Vallarta 10-320mg, Eviplera. The other patient is given Lescol 40 MG and Everolimus 1.5 mg tablet.""") ```

## Results ```bash | chunk | ner_label | ADE | RxNorm | Action | Treatment | UMLS | SNOMED_CT | NDC_Product | NDC_Package | |:-----------------------------|:------------|:----------------------------|---------:|:---------------------------|:-------------------------------------------|:---------|:------------|:--------------|:--------------| | Amlodopine Vallarta 10-320mg | DRUG | Gynaecomastia | 722131 | NONE | NONE | C1949334 | 425838008 | 00093-7693 | 00093-7693-56 | | Eviplera | DRUG | Anxiety | 217010 | Inhibitory Bone Resorption | Osteoporosis | C0720318 | NONE | NONE | NONE | | Lescol 40 MG | DRUG | NONE | 103919 | Hypocholesterolemic | Heterozygous Familial Hypercholesterolemia | C0353573 | NONE | 00078-0234 | 00078-0234-05 | | Everolimus 1.5 mg tablet | DRUG | Acute myocardial infarction | 2056895 | NONE | NONE | C4723581 | NONE | 00054-0604 | 00054-0604-21 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|medication_resolver_transform_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.1 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - TextMatcherModel - ChunkMergeModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger - Doc2Chunk - ResolverMerger - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - Doc2Chunk - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - Finisher --- layout: model title: English image_classifier_vit_rust_image_classification_1 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_1 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_1` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_1_en_4.1.0_3.0_1660166731268.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_1_en_4.1.0_3.0_1660166731268.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_1", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_1", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_1| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Niuean author: John Snow Labs name: opus_mt_en_niu date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, niu, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `niu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_niu_xx_2.7.0_2.4_1609167483288.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_niu_xx_2.7.0_2.4_1609167483288.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_niu| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xlsr_nahuatl TFWav2Vec2ForCTC from tyoc213 author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_nahuatl date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_nahuatl` is a English model originally trained by tyoc213. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_nahuatl_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_nahuatl_en_4.2.0_3.0_1664110207282.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_nahuatl_en_4.2.0_3.0_1664110207282.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_nahuatl', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_nahuatl", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_nahuatl| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast and Accurate Language Identification - 95 Languages (CNN) author: John Snow Labs name: ld_wiki_tatoeba_cnn_95 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Afrikaans`, `Amharic`, `Aragonese`, `Arabic`, `Assamese`, `Azerbaijani`, `Belarusian`, `Bulgarian`, `Bengali`, `Breton`, `Bosnian`, `Catalan`, `Czech`, `Welsh`, `Danish`, `German`, `Greek`, `English`, `Esperanto`, `Spanish`, `Estonian`, `Basque`, `Persian`, `Finnish`, `Faroese`, `French`, `Irish`, `Galician`, `Gujarati`, `Hebrew`, `Hindi`, `Croatian`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Indonesian`, `Icelandic`, `Italian`, `Japanese`, `Javanese`, `Georgian`, `Kazakh`, `Khmer`, `Kannada`, `Korean`, `Kurdish`, `Kyrgyz`, `Latin`, `Luxembourgish`, `Lao`, `Lithuanian`, `Latvian`, `Malagasy`, `Macedonian`, `Malayalam`, `Mongolian`, `Marathi`, `Malay`, `Maltese`, `Nepali`, `Dutch`, `Norwegian Nynorsk`, `Norwegian`, `Occitan`, `Odia (Oriya)`, `Punjabi (Eastern)`, `Polish`, `Pashto`, `Portuguese`, `Quechua`, `Romanian`, `Russian`, `Northern Sami`, `Sinhala`, `Slovak`, `Slovenian`, `Albanian`, `Serbian`, `Swedish`, `Swahili`, `Tamil`, `Telugu`, `Thai`, `Tagalog`, `Turkish`, `Tatar`, `Uyghur`, `Ukrainian`, `Urdu`, `Vietnamese`, `Volapük`, `Walloon`, `Xhosa`, `Chinese`. ## Predicted Entities `af`, `am`, `an`, `ar`, `as`, `az`, `be`, `bg`, `bn`, `br`, `bs`, `ca`, `cs`, `cy`, `da`, `de`, `el`, `en`, `eo`, `es`, `et`, `eu`, `fa`, `fi`, `fo`, `fr`, `ga`, `gl`, `gu`, `he`, `hi`, `hr`, `ht`, `hu`, `hy`, `ia`, `id`, `is`, `it`, `ja`, `jv`, `ka`, `kk`, `km`, `kn`, `ko`, `ku`, `ky`, `la`, `lb`, `lo`, `lt`, `lv`, `mg`, `mk`, `ml`, `mn`, `mr`, `ms`, `mt`, `ne`, `nl`, `nn`, `no`, `oc`, `or`, `pa`, `pl`, `ps`, `pt`, `qu`, `ro`, `ru`, `se`, `si`, `sk`, `sl`, `sq`, `sr`, `sv`, `sw`, `ta`, `te`, `th`, `tl`, `tr`, `tt`, `ug`, `uk`, `ur`, `vi`, `vo`, `wa`, `xh`, `zh`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_95_xx_2.7.0_2.4_1607184332861.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_95_xx_2.7.0_2.4_1607184332861.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_95", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_95", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_95').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_tatoeba_cnn_95| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia and Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | fr| 1000| 999| 0.999| | de| 1000| 998| 0.998| | fi| 1000| 998| 0.998| | pt| 1000| 996| 0.996| | sv| 1000| 995| 0.995| | el| 1000| 994| 0.994| | nl| 1000| 994| 0.994| | it| 1000| 994| 0.994| | en| 1000| 993| 0.993| | es| 1000| 984| 0.984| | hu| 880| 865|0.9829545454545454| | ro| 784| 769|0.9808673469387755| | lt| 1000| 978| 0.978| | et| 928| 906|0.9762931034482759| | cs| 1000| 975| 0.975| | pl| 914| 890| 0.973741794310722| | da| 1000| 958| 0.958| | sk| 1000| 947| 0.947| | bg| 1000| 939| 0.939| | lv| 916| 849|0.9268558951965066| | sl| 914| 844|0.9234135667396062| +--------+-----+-------+------------------+ +-------+-------------------+ |summary| precision| +-------+-------------------+ | count| 21| | mean| 0.9764822024804014| | stddev|0.02384830734809143| | min| 0.9234135667396062| | max| 0.999| +-------+-------------------+ ``` --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from jamarju) author: John Snow Labs name: roberta_qa_base_bne_squad_2.0 date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-squad-2.0-es` is a Spanish model originally trained by `jamarju`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_bne_squad_2.0_es_4.2.4_3.0_1669985918426.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_bne_squad_2.0_es_4.2.4_3.0_1669985918426.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_squad_2.0","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_bne_squad_2.0","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_bne_squad_2.0| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|456.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jamarju/roberta-base-bne-squad-2.0-es - https://github.com/PlanTL-SANIDAD/lm-spanish - https://github.com/ccasimiro88/TranslateAlignRetrieve --- layout: model title: English BertForQuestionAnswering Cased model (from CherylTSW) author: John Snow Labs name: bert_qa_cheryltsw_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `CherylTSW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_cheryltsw_finetuned_squad_en_4.0.0_3.0_1657186031141.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_cheryltsw_finetuned_squad_en_4.0.0_3.0_1657186031141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_cheryltsw_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_cheryltsw_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_cheryltsw_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/CherylTSW/bert-finetuned-squad --- layout: model title: English asr_bach_arb TFWav2Vec2ForCTC from bkh6722 author: John Snow Labs name: asr_bach_arb date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_bach_arb` is a English model originally trained by bkh6722. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_bach_arb_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_bach_arb_en_4.2.0_3.0_1664118857579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_bach_arb_en_4.2.0_3.0_1664118857579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_bach_arb", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_bach_arb", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_bach_arb| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|349.2 MB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from sabah17) author: John Snow Labs name: distilbert_qa_sabah17_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `sabah17`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sabah17_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772373637.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sabah17_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772373637.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sabah17_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sabah17_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sabah17_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sabah17/distilbert-base-uncased-finetuned-squad --- layout: model title: Persian BertForQuestionAnswering Base Uncased model (from aminnaghavi) author: John Snow Labs name: bert_qa_base_parsbert_uncased_finetuned_perqa date: 2022-07-07 tags: [fa, open_source, bert, question_answering] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-parsbert-uncased-finetuned-perQA` is a Persian model originally trained by `aminnaghavi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_parsbert_uncased_finetuned_perqa_fa_4.0.0_3.0_1657183345692.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_parsbert_uncased_finetuned_perqa_fa_4.0.0_3.0_1657183345692.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_parsbert_uncased_finetuned_perqa","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_parsbert_uncased_finetuned_perqa","fa") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_parsbert_uncased_finetuned_perqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aminnaghavi/bert-base-parsbert-uncased-finetuned-perQA --- layout: model title: Detect Family History Status from Oncology Entities author: John Snow Labs name: assertion_oncology_family_history_wip date: 2022-10-11 tags: [licensed, clinical, oncology, en, assertion, family_history] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects entities refering to the family history. ## Predicted Entities `Family_History`, `Other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_family_history_wip_en_4.0.0_3.0_1665522020132.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_family_history_wip_en_4.0.0_3.0_1665522020132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------------|:------------|:---------------| | breast cancer | Cancer_Dx | Family_History | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_family_history_wip| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Family_History 0.88 0.96 0.92 24.0 Other 0.96 0.90 0.93 29.0 macro-avg 0.92 0.93 0.92 53.0 weighted-avg 0.93 0.92 0.92 53.0 ``` --- layout: model title: Text Detection author: John Snow Labs name: image_text_detector_v2 date: 2023-01-10 tags: [en, licensed] task: OCR Text Detection & Recognition language: en nav_key: models edition: Visual NLP 4.1.0 spark_version: 3.2.1 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description CRAFT: Character-Region Awareness For Text detection, is designed with a convolutional neural network producing the character region score and affinity score. The region score is used to localize individual characters in the image, and the affinity score is used to group each character into a single instance. To compensate for the lack of character-level annotations, we propose a weaklysupervised learning framework that estimates characterlevel ground truths in existing real word-level datasets. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/ocr/TEXT_DETECTION/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-ocr-workshop/blob/master/jupyter/Cards/SparkOcrImageTextDetection.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/ocr/image_text_detector_v2_en_3.3.0_2.4_1643618928538.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/ocr/image_text_detector_v2_en_3.3.0_2.4_1643618928538.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python binary_to_image = BinaryToImage() \ .setInputCol("content") \ .setOutputCol("image") \ .setImageType(ImageType.TYPE_3BYTE_BGR) text_detector = ImageTextDetectorV2 \ .pretrained("image_text_detector_v2", "en", "clinical/ocr") \ .setInputCol("image") \ .setOutputCol("text_regions") \ .setScoreThreshold(0.5) \ .setTextThreshold(0.2) \ .setSizeThreshold(10) \ .setWithRefiner(True) draw_regions = ImageDrawRegions() \ .setInputCol("image") \ .setInputRegionsCol("text_regions") \ .setOutputCol("image_with_regions") \ .setRectColor(Color.green) \ .setRotated(True) pipeline = PipelineModel(stages=[ binary_to_image, text_detector, draw_regions ]) imagePath = pkg_resources.resource_filename('sparkocr', 'resources/ocr/text_detection/020_Yas_patella1.jpg') image_df = spark.read.format("binaryFile").load(imagePath).sort("path") result = pipeline.transform(image_df) ``` ```scala val binary_to_image = new BinaryToImage() .setInputCol("content") .setOutputCol("image") .setImageType(ImageType.TYPE_3BYTE_BGR) val text_detector = ImageTextDetectorV2 .pretrained("image_text_detector_v2", "en", "clinical/ocr") .setInputCol("image") .setOutputCol("text_regions") .setScoreThreshold(0.5) .setTextThreshold(0.2) .setSizeThreshold(10) .setWithRefiner(True) val draw_regions = new ImageDrawRegions() .setInputCol("image") .setInputRegionsCol("text_regions") .setOutputCol("image_with_regions") .setRectColor(Color.green) .setRotated(True) val pipeline = new PipelineModel().setStages(Array( binary_to_image, text_detector, draw_regions)) val imagePath = pkg_resources.resource_filename("sparkocr", "resources/ocr/text_detection/020_Yas_patella1.jpg") val image_df = spark.read.format("binaryFile").load(imagePath).sort("path") val result = pipeline.transform(image_df) ```

## Example {%- capture input_image -%} ![Screenshot](/assets/images/examples_ocr/image6.png) {%- endcapture -%} {%- capture output_image -%} ![Screenshot](/assets/images/examples_ocr/image6_out.png) {%- endcapture -%} {% include templates/input_output_image.md input_image=input_image output_image=output_image %} {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|text_detection_v2| |Type:|ocr| |Compatibility:|Visual NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Output Labels:|[text_regions]| |Language:|en| --- layout: model title: English BertForQuestionAnswering model (from andi611) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-squad2-with-ner-mit-restaurant-with-neg-with-repeat` is a English model orginally trained by `andi611`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat_en_4.0.0_3.0_1654537564717.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat_en_4.0.0_3.0_1654537564717.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.large_uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_squad2_with_ner_mit_restaurant_with_neg_with_repeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/andi611/bert-large-uncased-whole-word-masking-squad2-with-ner-mit-restaurant-with-neg-with-repeat --- layout: model title: BioBERT Embeddings (Pubmed Large) author: John Snow Labs name: biobert_pubmed_large_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pubmed_large_cased_en_2.6.0_2.4_1598342382907.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pubmed_large_cased_en_2.6.0_2.4_1598342382907.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash token en_embed_biobert_pubmed_large_cased_embeddings I [-0.041047871112823486, 0.24242812395095825, 0... hate [-0.6859451532363892, -0.45743268728256226, -0... cancer [-0.12403186410665512, 0.6688604354858398, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_pubmed_large_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: Legal Assigns Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_assigns_bert date: 2023-03-05 tags: [en, legal, classification, clauses, assigns, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Assigns` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Assigns`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_assigns_bert_en_1.0.0_3.0_1678050622215.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_assigns_bert_en_1.0.0_3.0_1678050622215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_assigns_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Assigns]| |[Other]| |[Other]| |[Assigns]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_assigns_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.9 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Assigns 0.83 1.00 0.91 5 Other 1.00 0.67 0.80 3 accuracy - - 0.88 8 macro-avg 0.92 0.83 0.85 8 weighted-avg 0.90 0.88 0.87 8 ``` --- layout: model title: Chinese Bert Embeddings (3-layer) author: John Snow Labs name: bert_embeddings_rbt3 date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `rbt3` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt3_zh_3.4.2_3.0_1649669074305.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt3_zh_3.4.2_3.0_1649669074305.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt3| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|144.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt3 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Clinical Deidentification author: John Snow Labs name: clinical_deidentification date: 2022-03-03 tags: [deidentification, pipeline, de, licensed,clinical] task: Pipeline Healthcare language: de edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from **German** medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `PATIENT`, `HOSPITAL`, `DATE`, `ORGANIZATION`, `CITY`, `STREET`, `USERNAME`, `PROFESSION`, `PHONE`, `COUNTRY`, `DOCTOR`, `AGE`, `CONTACT`, `ID`, `PHONE`, `ZIP`, `ACCOUNT`, `SSN`, `DLN`, `PLATE` entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_de_3.4.1_3.0_1646330183939.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_de_3.4.1_3.0_1646330183939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "de", "clinical/models") sample = """Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300 """ result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = PretrainedPipeline("clinical_deidentification","de","clinical/models") val sample = "Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("de.deid.clinical").predict("""Zusammenfassung : Michael Berger wird am Morgen des 12 Dezember 2018 ins St.Elisabeth Krankenhaus eingeliefert. Herr Michael Berger ist 76 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: T0110053F Platte A-BC124 Kontonummer: DE89370400440532013000 SSN : 13110587M565 Lizenznummer: B072RRE2I55 Adresse : St.Johann-Straße 13 19300 """) ```

## Results ```bash Masked with entity labels ------------------------------ Zusammenfassung : wird am Morgen des ins eingeliefert. Herr ist Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: Platte Kontonummer: SSN : Lizenznummer: Adresse : Masked with chars ------------------------------ Zusammenfassung : [************] wird am Morgen des [**************] ins [**********************] eingeliefert. Herr [************] ist ** Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: [*******] Platte [*****] Kontonummer: [********************] SSN : [**********] Lizenznummer: [*********] Adresse : [*****************] [***] Masked with fixed length chars ------------------------------ Zusammenfassung : **** wird am Morgen des **** ins **** eingeliefert. Herr **** ist **** Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: **** Platte **** Kontonummer: **** SSN : **** Lizenznummer: **** Adresse : **** **** Obfusceted ------------------------------ Zusammenfassung : Herrmann Kallert wird am Morgen des 11-26-1977 ins International Neuroscience eingeliefert. Herr Herrmann Kallert ist 79 Jahre alt und hat zu viel Wasser in den Beinen. Persönliche Daten : ID-Nummer: 136704D357 Platte QA348G Kontonummer: 192837465738 SSN : 1310011981M454 Lizenznummer: XX123456 Adresse : Klingelhöferring 31206 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|de| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ContextualParserModel - ChunkMergeModel - DeIdentificationModel - Finisher --- layout: model title: Detect former names of companies in texts (small) author: John Snow Labs name: finner_wiki_formername date: 2023-01-15 tags: [former, name, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model, aimed to detect Former Names of companies. It was trained with wikipedia texts about companies. ## Predicted Entities `FORMER_NAME`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_wiki_formername_en_1.0.0_3.0_1673797854205.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_wiki_formername_en_1.0.0_3.0_1673797854205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) chunks = finance.NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") ner = finance.NerModel().pretrained("finner_wiki_formername", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") pipe = nlp.Pipeline(stages=[documenter, sentencizer, tokenizer, embeddings, ner, chunks]) model = pipe.fit(df) res = model.transform(df) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.begin, res.ner_chunk.end, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['3']['sentence']").alias("sentence_id"), F.expr("cols['0']").alias("chunk"), F.expr("cols['2']").alias("end"), F.expr("cols['3']['entity']").alias("ner_label"))\ .filter("ner_label!='O'")\ .show(truncate=False) ```

## Results ```bash +-----------+------------------+---+-----------+ |sentence_id|chunk |end|ner_label | +-----------+------------------+---+-----------+ |0 |Toro Motor Company|57 |FORMER_NAME| +-----------+------------------+---+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_wiki_formername| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.2 MB| ## References Wikipedia ## Benchmarking ```bash label tp fp fn prec rec f1 I-FORMER_NAME 29 20 13 0.59183675 0.6904762 0.63736266 B-FORMER_NAME 19 5 8 0.7916667 0.7037037 0.7450981 Macro-average 48 25 21 0.6917517 0.6970899 0.69441056 Micro-average 48 25 21 0.65753424 0.6956522 0.6760564 ``` --- layout: model title: NER Pipeline for 9 African Languages author: John Snow Labs name: distilbert_base_token_classifier_masakhaner_pipeline date: 2022-06-25 tags: [hausa, igbo, kinyarwanda, luganda, nigerian, pidgin, swahilu, wolof, yoruba, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [distilbert_base_token_classifier_masakhaner](https://nlp.johnsnowlabs.com/2022/01/18/distilbert_base_token_classifier_masakhaner_xx.html) model. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/Ner_masakhaner/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Ner_masakhaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_token_classifier_masakhaner_pipeline_xx_4.0.0_3.0_1656118926686.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_token_classifier_masakhaner_pipeline_xx_4.0.0_3.0_1656118926686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python masakhaner_pipeline = PretrainedPipeline("distilbert_base_token_classifier_masakhaner_pipeline", lang = "xx") masakhaner_pipeline.annotate("Ilé-iṣẹ́ẹ Mohammed Sani Musa, Activate Technologies Limited, ni ó kó ẹ̀rọ Ìwé-pélébé Ìdìbò Alálòpẹ́ (PVCs) tí a lò fún ìbò ọdún-un 2019, ígbà tí ó jẹ́ òǹdíjedupò lábẹ́ ẹgbẹ́ olóṣèlúu tí ó ń tukọ̀ ètò ìṣèlú lọ́wọ́ All rogressives Congress (APC) fún Aṣojú Ìlà-Oòrùn Niger, ìyẹn gẹ́gẹ́ bí ilé iṣẹ́ aṣèwádìí, Premium Times ṣe tẹ̀ ẹ́ jáde.") ``` ```scala val masakhaner_pipeline = new PretrainedPipeline("distilbert_base_token_classifier_masakhaner_pipeline", lang = "xx") masakhaner_pipeline.annotate("Ilé-iṣẹ́ẹ Mohammed Sani Musa, Activate Technologies Limited, ni ó kó ẹ̀rọ Ìwé-pélébé Ìdìbò Alálòpẹ́ (PVCs) tí a lò fún ìbò ọdún-un 2019, ígbà tí ó jẹ́ òǹdíjedupò lábẹ́ ẹgbẹ́ olóṣèlúu tí ó ń tukọ̀ ètò ìṣèlú lọ́wọ́ All rogressives Congress (APC) fún Aṣojú Ìlà-Oòrùn Niger, ìyẹn gẹ́gẹ́ bí ilé iṣẹ́ aṣèwádìí, Premium Times ṣe tẹ̀ ẹ́ jáde.") ```

## Results ```bash +-----------------------------+---------+ |chunk |ner_label| +-----------------------------+---------+ |Mohammed Sani Musa |PER | |Activate Technologies Limited|ORG | |ọdún-un 2019 |DATE | |All rogressives Congress |ORG | |APC |ORG | |Aṣojú Ìlà-Oòrùn Niger |LOC | |Premium Times |ORG | +-----------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_token_classifier_masakhaner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|xx| |Size:|505.8 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - DistilBertForTokenClassification - NerConverter - Finisher --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13 TFWav2Vec2ForCTC from PereLluis13 author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13` is a Modern Greek (1453-) model originally trained by PereLluis13. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13_el_4.2.0_3.0_1664106724119.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13_el_4.2.0_3.0_1664106724119.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13", "el")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13", "el") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_greek_by_PereLluis13| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|el| |Size:|1.2 GB| --- layout: model title: Recognize Entities DL Pipeline for Norwegian (Bokmal) - Small author: John Snow Labs name: entity_recognizer_sm date: 2021-03-22 tags: [open_source, norwegian_bokmal, entity_recognizer_sm, pipeline, "no"] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: "no" edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The entity_recognizer_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_no_3.0.0_3.0_1616442836782.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/entity_recognizer_sm_no_3.0.0_3.0_1616442836782.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('entity_recognizer_sm', lang = 'no') annotations = pipeline.fullAnnotate(""Hei fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("entity_recognizer_sm", lang = "no") val result = pipeline.fullAnnotate("Hei fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei fra John Snow Labs! ""] result_df = nlu.load('no.ner').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hei fra John Snow Labs! '] | ['Hei fra John Snow Labs!'] | ['Hei', 'fra', 'John', 'Snow', 'Labs!'] | [[-0.394499987363815,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|entity_recognizer_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|no| --- layout: model title: English asr_wav2vec2_indian_english TFWav2Vec2ForCTC from anjulRajendraSharma author: John Snow Labs name: asr_wav2vec2_indian_english date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_indian_english` is a English model originally trained by anjulRajendraSharma. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_indian_english_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_indian_english_en_4.2.0_3.0_1664123154208.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_indian_english_en_4.2.0_3.0_1664123154208.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_indian_english", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_indian_english", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_indian_english| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.5 MB| --- layout: model title: Chunk Entity Resolver for billable ICD10-CM HCC codes author: John Snow Labs name: chunkresolve_icd10cm_hcc_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using chunk embeddings (augmented with synonyms, four times richer than previous resolver). It also adds support of 7-digit codes with HCC status. For reference: http://www.formativhealth.com/wp-content/uploads/2018/06/HCC-White-Paper.pdf ## Predicted Entities Outputs 7-digit billable ICD codes. In the result, look for `aux_label` parameter in the metadata to get HCC status. The HCC status can be divided to get further information: `billable status`, `hcc status`, and `hcc score`. For example, in the example shared below the `billable status is 1`, `hcc status is 1`, and `hcc score is 8`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_hcc_clinical_en_3.0.0_3.0_1617356679231.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_hcc_clinical_en_3.0.0_3.0_1617356679231.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_hcc_clinical","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") pipeline = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, resolver]) data = spark.createDataFrame([["metastatic lung cancer"]]).toDF("text") model = pipeline.fit(data) results = model.transform(data) ... ``` ```scala ... val resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_hcc_clinical","en","clinical/models") .setInputCols("token","chunk_embeddings") .setOutputCol("entity") val pipeline = new Pipeline().setStages(Array(document_assembler, sbert_embedder, resolver)) val data = Seq("metastatic lung cancer").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash | | chunks | code | resolutions | all_codes | billable_hcc_status_score | all_distances | |---:|:-----------------------|:-------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|:----------------------------|:-------------------------------------------------------------------------------------------------------------------------| | 0 | metastatic lung cancer | C7800 | ['cancer metastatic to lung', 'metastasis from malignant tumor of lung', 'cancer metastatic to left lung', 'history of cancer metastatic to lung', 'metastatic cancer', 'history of cancer metastatic to lung (situation)', 'metastatic adenocarcinoma to bilateral lungs', 'cancer metastatic to chest wall', 'metastatic malignant neoplasm to left lower lobe of lung', 'metastatic carcinoid tumour', 'cancer metastatic to respiratory tract', 'metastatic carcinoid tumor'] | ['C7800', 'C349', 'C7801', 'Z858', 'C800', 'Z8511', 'C780', 'C798', 'C7802', 'C799', 'C7830', 'C7B00'] | ['1', '1', '8'] | ['0.0464', '0.0829', '0.0852', '0.0860', '0.0914', '0.0989', '0.1133', '0.1220', '0.1220', '0.1253', '0.1249', '0.1260'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_icd10cm_hcc_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab4 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab4 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab4` is a English model originally trained by sameearif88. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab4_en_4.2.0_3.0_1664019107930.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab4_en_4.2.0_3.0_1664019107930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab4", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab4", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab4| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from nsridhar) author: John Snow Labs name: roberta_qa_finetuned_country date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-finetuned-country` is a English model originally trained by `nsridhar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_country_en_4.3.0_3.0_1674220262882.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_finetuned_country_en_4.3.0_3.0_1674220262882.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_country","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_finetuned_country","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_finetuned_country| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nsridhar/roberta-finetuned-country --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Anogmeld) author: John Snow Labs name: distilbert_qa_anogmeld_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Anogmeld`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_anogmeld_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768254089.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_anogmeld_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768254089.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anogmeld_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_anogmeld_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_anogmeld_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Anogmeld/distilbert-base-uncased-finetuned-squad --- layout: model title: Kinyarwanda XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand date: 2022-08-01 tags: [rw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: rw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-swahili-finetuned-ner-kinyarwanda` is a Kinyarwanda model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659355664108.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659355664108.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand","rw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand","rw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_swahili_finetuned_ner_kinyarwand| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|rw| |Size:|1.0 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-swahili-finetuned-ner-kinyarwanda - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: Clinical Deidentification Pipeline (English) author: John Snow Labs name: clinical_deidentification date: 2022-07-24 tags: [deidentification, deid, clinical, pipeline, en, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate `AGE`, `CONTACT`, `DATE`, `ID`, `LOCATION`, `NAME`, `PROFESSION`, `CITY`, `COUNTRY`, `DOCTOR`, `HOSPITAL`, `IDNUM`, `MEDICALRECORD`, `ORGANIZATION`, `PATIENT`, `PHONE`, `PROFESSION`, `STREET`, `USERNAME`, `ZIP`, `ACCOUNT`, `LICENSE`, `VIN`, `SSN`, `DLN`, `PLATE`, `IPADDR`, `EMAIL` entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT_MULTI/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT_MULTI.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_4.0.0_3.0_1658693448275.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_en_4.0.0_3.0_1658693448275.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "en", "clinical/models") sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" result = deid_pipeline.annotate(sample) print("\n".join(result['masked'])) print("\n".join(result['masked_with_chars'])) print("\n".join(result['masked_fixed_length_chars'])) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification","en","clinical/models") val sample = """Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""" val result = deid_pipeline.annotate(sample) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.clinical_pipeline").predict("""Name : Hendrickson, Ora, Record date: 2093-01-13, # 719435. Dr. John Green, ID: 1231511863, IP 203.120.223.13. He is a 60-year-old male was admitted to the Day Hospital for cystectomy on 01/13/93. Patient's VIN : 1HGBH41JXMN109286, SSN #333-44-6666, Driver's license no:A334455B. Phone (302) 786-5227, 0295 Keats Street, San Francisco, E-MAIL: smith@gmail.com.""") ```

?/*+` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/1.SparkNLP_Basics.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/match_chunks_en_3.3.4_3.0_1641307675339.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/match_chunks_en_3.3.4_3.0_1641307675339.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['the restaurant yesterday', 'family', 'the day', 'that time', 'today', 'the food', 'tomorrow'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|match_chunks| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|4.1 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - Chunker --- layout: model title: Sentence Entity Resolver for Snomed Aux Concepts, INT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_auxConcepts date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts to Snomed codes codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. This model is capable of extracting Morph Abnormality, Procedure, Substance, Physical Object, and Body Structure concepts of Snomed codes. It has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts Snomed Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_en_3.0.4_3.0_1621189567327.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_auxConcepts_en_3.0.4_3.0_1621189567327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") chunk2doc = Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") snomed_aux_int_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_snomed_auxConcepts","en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_int_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val chunk2doc = new Chunk2Doc() .setInputCols("ner_chunk") .setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols("ner_chunk_doc") .setOutputCol("sbert_embeddings") val snomed_aux_int_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_snomed_auxConcepts","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, snomed_aux_int_resolver)) val data = Seq("""This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.snomed").predict("""This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .""") ```

## Results ```bash +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 148439002| 0.2138|risk factors pres...|148439002:::42595...| |chronic renal ins...| 83|109| PROBLEM| 722403003| 0.8517|gastrointestinal ...|722403003:::13781...| | COPD| 113|116| PROBLEM|845101000000100| 0.0962|management of chr...|845101000000100::...| | gastritis| 120|128| PROBLEM| 711498001| 0.3398|magnetic resonanc...|711498001:::71771...| | TIA| 136|138| PROBLEM| 449758002| 0.1927|traumatic infarct...|449758002:::85844...| |a non-ST elevatio...| 182|202| PROBLEM| 1411000087101| 0.0823|ct of left knee::...|1411000087101:::3...| |Guaiac positive s...| 208|229| PROBLEM| 388507006| 0.0555|asparagus rast:::...|388507006:::71771...| |cardiac catheteri...| 295|317| TEST| 41976001| 0.9790|cardiac catheteri...|41976001:::705921...| | PTCA| 324|327|TREATMENT| 312644004| 0.0616|angioplasty of po...|312644004:::41507...| | mid LAD lesion| 332|345| PROBLEM| 91749005| 0.1399|structure of firs...|91749005:::917470...| +--------------------+-----+---+---------+---------------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_auxConcepts| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[snomed_code_ct_aux_loaded]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on SNOMED (INT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: Multilingual BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_en_hi_dev date: 2022-07-07 tags: [xx, open_source, bert, question_answering] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-en-hi` is a Multilingual model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_en_hi_dev_xx_4.0.0_3.0_1657189998034.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_en_hi_dev_xx_4.0.0_3.0_1657189998034.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_en_hi_dev","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_en_hi_dev","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_en_hi_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-en-hi --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from it5) author: John Snow Labs name: t5_it5_efficient_small_el32_news_summarization date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-el32-news-summarization` is a Italian model originally trained by `it5`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_news_summarization_it_4.3.0_3.0_1675103478274.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_news_summarization_it_4.3.0_3.0_1675103478274.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_news_summarization","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_news_summarization","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_el32_news_summarization| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|594.0 MB| ## References - https://huggingface.co/it5/it5-efficient-small-el32-news-summarization - https://github.com/stefan-it - https://arxiv.org/abs/2203.03759 - https://gsarti.com - https://malvinanissim.github.io - https://arxiv.org/abs/2109.10686 - https://github.com/gsarti/it5 - https://paperswithcode.com/sota?task=News+Summarization&dataset=NewsSum-IT --- layout: model title: Detect Clinical Entities (ner_jsl) author: John Snow Labs name: ner_jsl date: 2021-04-30 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. Definitions of Predicted Entities: - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Death_Entity`: Mentions that indicate the death of a patient. - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). - `Drug_Ingredient`: Active ingredient/s found in drug products. - `Weight`: All mentions related to a patients weight. - `Drug_BrandName`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Gender`: Gender-specific nouns and pronouns. - `Temperature`: All mentions that refer to body temperature. - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Respiration`: Number of breaths per minute. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Frequency`: Frequency of administration for a dose prescribed. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Allergen`: Allergen related extractions mentioned in the document. ## Predicted Entities `Diagnosis` , `Procedure_Name` , `Lab_Result` , `Procedure` , `Procedure_Findings` , `O2_Saturation` , `Procedure_incident_description` , `Dosage` , `Causative_Agents_(Virus_and_Bacteria)` , `Name` , `Cause_of_death` , `Substance_Name` , `Weight` , `Symptom_Name` , `Maybe` , `Modifier` , `Blood_Pressure` , `Frequency` , `Gender` , `Drug_incident_description` , `Age` , `Drug_Name` , `Temperature` , `Section_Name` , `Route` , `Negation` , `Negated` , `Allergenic_substance` , `Lab_Name` , `Respiratory_Rate` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_en_3.0.0_3.0_1619768531594.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_en_3.0.0_3.0_1619768531594.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------+------------+ |chunk |ner | +---------------------------+------------+ |21-day-old |Age | |male |Gender | |congestion |Symptom_Name| |mom |Gender | |suctioning yellow discharge|Symptom_Name| |she |Gender | |mild |Modifier | |problems with his breathing|Symptom_Name| |negative |Negated | |perioral cyanosis |Symptom_Name| |retractions |Symptom_Name| |mom |Gender | |Tylenol |Drug_Name | |His |Gender | |his |Gender | |respiratory congestion |Symptom_Name| |He |Gender | |tired |Symptom_Name| |fussy |Symptom_Name| |albuterol |Drug_Name | +---------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash label tp fp fn prec rec f1 B-Pulse_Rate 77 39 12 0.663793 0.865169 0.75122 I-Diagnosis 2134 1139 1329 0.652001 0.616229 0.63361 I-Procedure_Name 2335 1329 956 0.637282 0.709511 0.671459 B-Lab_Result 601 182 94 0.767561 0.864748 0.813261 B-Procedure 1 0 5 1 0.166667 0.285714 B-Procedure_Findings 2 13 72 0.133333 0.027027 0.044944 B-O2_Saturation 1 3 4 0.25 0.2 0.222222 B-Dosage 477 197 68 0.707715 0.875229 0.782609 I-Causative_Agents_(Virus_and_Bacteria) 12 2 7 0.857143 0.631579 0.727273 B-Name 562 268 554 0.677108 0.503584 0.577595 I-Cause_of_death 9 5 11 0.642857 0.45 0.529412 I-Substance_Name 24 34 54 0.413793 0.307692 0.352941 I-Name 716 377 710 0.655078 0.502104 0.56848 B-Cause_of_death 9 6 8 0.6 0.529412 0.5625 B-Weight 52 22 9 0.702703 0.852459 0.77037 B-Symptom_Name 4364 1916 1652 0.694904 0.725399 0.709824 I-Maybe 27 51 61 0.346154 0.306818 0.325301 I-Symptom_Name 2073 1492 2348 0.581487 0.468898 0.519159 B-Modifier 1573 890 768 0.638652 0.671935 0.654871 B-Blood_Pressure 76 19 13 0.8 0.853933 0.826087 B-Frequency 308 134 77 0.696833 0.8 0.744861 I-Gender 26 31 28 0.45614 0.481482 0.468468 I-Drug_incident_description 4 10 57 0.285714 0.0655738 0.106667 B-Drug_incident_description 2 5 23 0.285714 0.08 0.125 I-Age 5 0 9 1 0.357143 0.526316 B-Drug_Name 1741 490 290 0.780368 0.857213 0.816987 B-Substance_Name 148 41 48 0.783069 0.755102 0.768831 B-Temperature 56 23 13 0.708861 0.811594 0.756757 I-Procedure 1 0 7 1 0.125 0.222222 B-Section_Name 2711 317 166 0.89531 0.942301 0.918205 I-Route 119 110 189 0.519651 0.386364 0.443203 B-Maybe 143 80 127 0.641256 0.52963 0.580122 B-Gender 5166 709 58 0.879319 0.988897 0.930895 I-Dosage 434 196 87 0.688889 0.833013 0.754127 B-Causative_Agents_(Virus_and_Bacteria) 19 3 8 0.863636 0.703704 0.77551 I-Frequency 275 134 191 0.672372 0.590129 0.628571 B-Age 357 27 16 0.929688 0.957105 0.943197 I-Lab_Result 45 78 152 0.365854 0.228426 0.28125 B-Negation 99 38 38 0.722628 0.722628 0.722628 B-Diagnosis 2786 1342 913 0.674903 0.753177 0.711895 I-Section_Name 3885 1353 179 0.741695 0.955955 0.835304 B-Route 421 217 166 0.659875 0.717206 0.687347 I-Negation 11 30 24 0.268293 0.314286 0.289474 B-Procedure_Name 1490 811 522 0.647545 0.740557 0.690934 B-Negated 1490 332 215 0.817783 0.8739 0.844911 I-Allergenic_substance 1 0 12 1 0.0769231 0.142857 I-Negated 89 132 146 0.402715 0.378723 0.390351 I-Procedure_Findings 2 31 283 0.0606061 0.00701754 0.012579 B-Allergenic_substance 72 29 24 0.712871 0.75 0.730965 I-Weight 47 35 16 0.573171 0.746032 0.648276 B-Lab_Name 804 290 122 0.734918 0.868251 0.79604 I-Modifier 99 73 422 0.575581 0.190019 0.285714 I-Temperature 1 0 14 1 0.0666667 0.125 I-Drug_Name 362 284 261 0.560372 0.581059 0.570528 I-Lab_Name 284 194 127 0.594142 0.690998 0.63892 B-Respiratory_Rate 46 5 5 0.901961 0.901961 0.901961 Macro-average 38674 15571 13819 0.589085 0.515426 0.5498 Micro-average 38674 15571 13819 0.712951 0.736746 0.724653 ``` --- layout: model title: Legal Due Authorization Clause Binary Classifier author: John Snow Labs name: legclf_due_authorization_clause date: 2023-01-29 tags: [en, legal, classification, tax, treatment, clauses, due_authorization, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `due-authorization` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `due-authorization`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_due_authorization_clause_en_1.0.0_3.0_1674993500619.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_due_authorization_clause_en_1.0.0_3.0_1674993500619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_due_authorization_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[due-authorization]| |[other]| |[other]| |[due-authorization]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_due_authorization_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support due-authorization 0.98 1.00 0.99 61 other 1.00 0.99 1.00 106 accuracy - - 0.99 167 macro-avg 0.99 1.00 0.99 167 weighted-avg 0.99 0.99 0.99 167 ``` --- layout: model title: Ukrainian RoBERTa Embeddings author: John Snow Labs name: roberta_embeddings_ukr_roberta_base date: 2022-04-14 tags: [roberta, embeddings, uk, open_source] task: Embeddings language: uk edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `ukr-roberta-base` is a Ukrainian model orginally trained by `youscan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_ukr_roberta_base_uk_3.4.2_3.0_1649948817339.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_ukr_roberta_base_uk_3.4.2_3.0_1649948817339.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_ukr_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|uk| |Size:|474.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/youscan/ukr-roberta-base - https://dumps.wikimedia.org/ukwiki/latest/ukwiki-latest-pages-articles.xml.bz2 - https://oscar-public.huma-num.fr/shuffled/uk_dedup.txt.gz - https://github.com/youscan/language-models - https://twitter.com/vitaliradchenko --- layout: model title: Detect Chemical Compounds and Genes (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_chemprot date: 2021-10-19 tags: [berfortokenclassification, ner, chemprot, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.0 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect chemical compounds and genes in the medical text using the pretrained NER model. This model is trained with the `BertForTokenClassification` method from the `transformers` library and imported into Spark NLP. ## Predicted Entities `CHEMICAL`, `GENE-Y`, `GENE-N` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMPROT_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_CHEMPROT_CLINICAL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_en_3.3.0_2.4_1634644903577.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_en_3.3.0_2.4_1634644903577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |Keratinocyte growth factor |GENE-Y | |acidic fibroblast growth factor|GENE-Y | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_chemprot| |Compatibility:|Healthcare NLP 3.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|true| |Max sentense length:|512| ## Data Source This model is trained on a [ChemProt corpus](https://biocreative.bioinformatics.udel.edu/). ## Benchmarking ```bash label precision recall f1-score support B-CHEMICAL 0.93 0.79 0.85 8649 B-GENE-N 0.63 0.56 0.59 2752 B-GENE-Y 0.82 0.73 0.77 5490 I-CHEMICAL 0.90 0.79 0.84 1313 I-GENE-N 0.72 0.62 0.67 1993 I-GENE-Y 0.81 0.72 0.77 2420 accuracy - - 0.73 22617 macro-avg 0.75 0.74 0.75 22617 weighted-avg 0.83 0.73 0.78 22617 ``` --- layout: model title: Legal Topic Classification on Greek Legislation author: John Snow Labs name: legclf_legal_code date: 2023-05-12 tags: [el, legal, classification, bert, licensed, tensorflow] task: Text Classification language: el edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Greek Legal Code (GLC) is a dataset consisting of approx. 47k legal resources from Greek legislation. The origin of GLC is “Permanent Greek Legislation Code - Raptarchis”, a collection of Greek legislative documents classified into multi-level (from broader to more specialized) categories. Given the text of a document, the `legclf_legal_code` model predicts the corresponding class ## Predicted Entities `ΒΙΟΜΗΧΑΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΝΟΜΟΘΕΣΙΑ_ΔΗΜΩΝ_ΚΑΙ_ΚΟΙΝΟΤΗΤΩΝ`, `ΕΚΠΑΙΔΕΥΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΕΘΝΙΚΗ_ΑΜΥΝΑ`, `ΕΚΚΛΗΣΙΑΣΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΕΘΝΙΚΗ_ΟΙΚΟΝΟΜΙΑ`, `ΔΗΜΟΣΙΟ_ΛΟΓΙΣΤΙΚΟ`, `ΕΜΠΟΡΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΣΤΡΑΤΟΣ_ΞΗΡΑΣ`, `ΔΗΜΟΣΙΟΙ_ΥΠΑΛΛΗΛΟΙ`, `ΔΙΠΛΩΜΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΤΕΛΩΝΕΙΑΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΤΑΧΥΔΡΟΜΕΙΑ_ΤΗΛΕΠΙΚΟΙΝΩΝΙΕΣ`, `ΠΟΛΕΜΙΚΟ_ΝΑΥΤΙΚΟ`, `ΑΣΦΑΛΙΣΤΙΚΑ_ΤΑΜΕΙΑ`, `ΓΕΩΡΓΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΑΓΡΟΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΑΣΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΠΟΛΙΤΙΚΗ_ΔΙΚΟΝΟΜΙΑ`, `ΚΟΙΝΩΝΙΚΕΣ_ΑΣΦΑΛΙΣΕΙΣ`, `ΟΙΚΟΝΟΜΙΚΗ_ΔΙΟΙΚΗΣΗ`, `ΔΙΟΙΚΗΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΠΟΛΕΜΙΚΗ_ΑΕΡΟΠΟΡΙΑ`, `ΔΑΣΗ_ΚΑΙ_ΚΤΗΝΟΤΡΟΦΙΑ`, `ΑΓΟΡΑΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΝΟΜΟΘΕΣΙΑ_ΕΠΙΜΕΛΗΤΗΡΙΩΝ_ΣΥΝΕΤΑΙΡΙΣΜΩΝ_ΚΑΙ_ΣΩΜΑΤΕΙΩΝ`, `ΥΓΕΙΟΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΔΙΟΙΚΗΣΗ_ΔΙΚΑΙΟΣΥΝΗΣ`, `ΣΥΓΚΟΙΝΩΝΙΕΣ`, `ΠΟΛΙΤΙΚΗ_ΑΕΡΟΠΟΡΙΑ`, `ΕΠΙΣΤΗΜΕΣ_ΚΑΙ_ΤΕΧΝΕΣ`, `ΡΑΔΙΟΦΩΝΙΑ_ΚΑΙ_ΤΥΠΟΣ`, `ΕΡΓΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΛΙΜΕΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΣΥΝΤΑΓΜΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΑΜΕΣΗ_ΦΟΡΟΛΟΓΙΑ`, `ΑΣΤΥΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΔΗΜΟΣΙΑ_ΕΡΓΑ`, `ΝΟΜΙΚΑ_ΠΡΟΣΩΠΑ_ΔΗΜΟΣΙΟΥ_ΔΙΚΑΙΟΥ`, `ΠΟΙΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ`, `ΤΥΠΟΣ_ΚΑΙ_ΤΟΥΡΙΣΜΟΣ`, `ΚΟΙΝΩΝΙΚΗ_ΠΡΟΝΟΙΑ`, `ΕΜΠΟΡΙΚΗ_ΝΑΥΤΙΛΙΑ`, `ΠΕΡΙΟΥΣΙΑ_ΔΗΜΟΣΙΟΥ_ΚΑΙ_ΝΟΜΙΣΜΑ`, `ΕΜΜΕΣΗ_ΦΟΡΟΛΟΓΙΑ`, `ΕΛΕΓΚΤΙΚΟ_ΣΥΝΕΔΡΙΟ_ΚΑΙ_ΣΥΝΤΑΞΕΙΣ`, `ΝΟΜΟΘΕΣΙΑ_ΑΝΩΝΥΜΩΝ_ΕΤΑΙΡΕΙΩΝ_ΤΡΑΠΕΖΩΝ_ΚΑΙ_ΧΡΗΜΑΤΙΣΤΗΡΙΩΝ` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_legal_code_el_1.0.0_3.0_1683904327601.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_legal_code_el_1.0.0_3.0_1683904327601.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") sequence_classifier = legal.BertForSequenceClassification.pretrained("legclf_legal_code", "el", "legal/models")\ .setInputCols(["document","token"])\ .setOutputCol("class")\ .setCaseSensitive(True)\ .setMaxSentenceLength(512) clf_pipeline = nlp.Pipeline(stages=[ document_assembler, tokenizer, sequence_classifier ]) empty_df = spark.createDataFrame([['']]).toDF("text") model = clf_pipeline.fit(empty_df) text = """14. ΠΡΟΕΔΡΙΚΟΝ ΔΙΑΤΑΓΜΑ υπ’ αριθ. 545 της 5/25 Ιουλ. 1979 (ΦΕΚ Α΄ 168) Περί αυξήσεως των υπό του Ταμείου Προνοίας Εργοληπτών Δημοσίων Έργων παρεχομένων εφ’ άπαξ βοηθημάτων. Έχοντες υπ’ όψιν: 1.Τας διατάξεις της παρ. 8 του άρθρ. 2 του Ν.Δ. 75/1946 «περί συστάσεως Ταμείου Προνοίας Εργοληπτών Δημοσίων Έργων». 2.Τας διατάξεις της παρ. 2 του άρθρ. 12 του Νόμ. 400/1976 «περί Υπουργικού Συμβουλίου και Υπουργείων» (ΦΕΚ 203/76 τ.Α΄). 3.Τας διατάξεις των άρθρ. 17 παρ. 2 εδάφ. β΄ περίπτ. αα΄ και 113 παρ. 2 εδάφ. α΄ του Π.Δ. 544/1977 (ΦΕΚ 178/77 τ.Α΄) ως η τελευταία αντικατεστάθη δια της παρ. 1 του άρθρ. 2 του Νόμ. 728/1977 (ΦΕΚ 316/77 τ.Α΄). 4.Την υπ’ αριθ. Δ3/2087/6.12.77 (ΦΕΚ 1278/77 τ.Β΄) απόφασιν του Πρωθυπουργού και του Υπουργού Κοινωνικών Υπηρεσιών «περί αναθέσεως αρμοδιοτήτων στους Υφυπουργούς Κοινωνικών Υπηρεσιών». 5.Την σύμφωνον γνώμην του Διοικητικού Συμβουλίου του Ταμείου Προνοίας Εργοληπτών Δημοσίων Έργων, ληφθείσα κατά την υπ’ αριθ. 1/17.1.79 συνεδρίασιν αυτού και υποβληθείσα ημίν δια της υπ’ αριθ. 583/24.1.79 αναφοράς του Ταμείου. 6.Την γνωμοδότησιν του Συμβουλίου Κοινων. Ασφαλείας, ληφθείσα κατά την υπ’ αριθ. 9/11.4.79 συνεδρίασιν αυτού της Κ΄ περιόδου. 7.Την υπ’ αριθ. 490/1979 γνωμοδότησιν του Συμβουλίου της Επικρατείας, προτάσει του επί των Κοινωνικών Υπηρεσιών Υφυπουργού, αποφασίζομεν: Άρθρον μόνον.-1.Το υπό του Ταμείου Προνοίας Εργοληπτών Δημοσίων Έργων παρεχόμενον εις τους εξερχομένους του επαγγέλματος δι’ οιονδήποτε λόγον ησφαλισμένος αυτού, πλήρες εφ’ άπαξ βοήθημα (χορηγία) δια τους έχοντας συμπεπληρωμένην 35ετή υπηρεσίαν, καθορίζεται εφ’ εξής ως κάτωθι: α)Δια τους ησφαλισμένους Εργολήπτας Δημοσίων Έργων Γ΄ και Δ΄ τάξεως ως και τους μετόχους υπαλλήλους του Ταμείου και των Εργοληπτικών Οργανώσεων από του 6ου βαθμού συμπεριλαμβανομένου και άνω εις 474.500 δραχμάς. β)Δια τους λοιπούς ησφαλισμένους του Ταμείου εις 357.000 δραχμάς. 2.Εις περίπτωσιν κατά την οποίαν η συνολική υπηρεσία οιουδήποτε εκ των ανωτέρω ησφαλισμένων είναι μικροτέρα των 35 ετών και εφ’ όσον συντρέχουν αι προϋποθέσεις του άρθρ. 1 του Β.Δ/τος της 13/29 Μαρτ. 1947 «περί χορηγιών του Ταμείου Προνοίας Εργοληπτών Δημοσίων Έργων» ως ισχύει κατόπιν των τροποποιήσεων και συμπληρώσεων αυτού, ο ησφαλισμένος δικαιούται τόσων τριακοστών πέμπτων του πλήρους εφ’ άπαξ βοηθήματος, όσα και τα έτη της υπηρεσίας αυτού. 3.Προϋπηρεσία πλέον των 35 ετών δεν αναγνωρίζεται. (Αντί για τη σελ. 224,03(α) Σελ. 224,03(β) Τεύχος 709-Σελ. 113 Ταμείο Προνοίας Εργοληπτών 23.Γ.ε.12-14 Εις τον επί των Κοινωνικών Υπηρεσιών Υφυπουργόν ανατίθεμεν την δημοσίευσιν και εκτέλεσιν του παρόντος.""" result = model.transform(spark.createDataFrame([[text]]).toDF("text")) ```

## Results ```bash +--------------------+--------------+ | text| result| +--------------------+--------------+ |14. ΠΡΟΕΔΡΙΚΟΝ ΔΙ...|[ΔΗΜΟΣΙΑ_ΕΡΓΑ]| +--------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_legal_code| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|el| |Size:|356.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References The dataset is available [here](https://huggingface.co/datasets/greek_legal_code) ## Benchmarking ```bash label precision recall f1-score support ΑΓΟΡΑΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.79 0.83 0.81 41 ΑΓΡΟΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.84 0.87 0.85 30 ΑΜΕΣΗ_ΦΟΡΟΛΟΓΙΑ 0.87 0.90 0.89 94 ΑΣΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.81 0.83 0.82 41 ΑΣΤΥΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.85 0.87 0.86 70 ΑΣΦΑΛΙΣΤΙΚΑ_ΤΑΜΕΙΑ 0.93 0.95 0.94 121 ΒΙΟΜΗΧΑΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.85 0.83 0.84 93 ΓΕΩΡΓΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.88 0.92 0.90 155 ΔΑΣΗ_ΚΑΙ_ΚΤΗΝΟΤΡΟΦΙΑ 0.88 0.70 0.78 40 ΔΗΜΟΣΙΑ_ΕΡΓΑ 0.88 0.88 0.88 111 ΔΗΜΟΣΙΟ_ΛΟΓΙΣΤΙΚΟ 0.79 0.74 0.76 46 ΔΗΜΟΣΙΟΙ_ΥΠΑΛΛΗΛΟΙ 0.78 0.79 0.78 90 ΔΙΟΙΚΗΣΗ_ΔΙΚΑΙΟΣΥΝΗΣ 0.91 0.97 0.94 140 ΔΙΟΙΚΗΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.80 0.86 0.83 42 ΔΙΠΛΩΜΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.82 0.87 0.85 95 ΕΘΝΙΚΗ_ΑΜΥΝΑ 0.84 0.80 0.82 121 ΕΘΝΙΚΗ_ΟΙΚΟΝΟΜΙΑ 0.76 0.79 0.77 43 ΕΚΚΛΗΣΙΑΣΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.92 0.98 0.95 47 ΕΚΠΑΙΔΕΥΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.94 0.94 0.94 230 ΕΛΕΓΚΤΙΚΟ_ΣΥΝΕΔΡΙΟ_ΚΑΙ_ΣΥΝΤΑΞΕΙΣ 0.79 0.70 0.74 37 ΕΜΜΕΣΗ_ΦΟΡΟΛΟΓΙΑ 0.81 0.84 0.83 62 ΕΜΠΟΡΙΚΗ_ΝΑΥΤΙΛΙΑ 0.95 0.95 0.95 149 ΕΜΠΟΡΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.85 0.95 0.90 42 ΕΠΙΣΤΗΜΕΣ_ΚΑΙ_ΤΕΧΝΕΣ 0.94 0.95 0.95 285 ΕΡΓΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.90 0.88 0.89 95 ΚΟΙΝΩΝΙΚΕΣ_ΑΣΦΑΛΙΣΕΙΣ 0.92 0.89 0.91 66 ΚΟΙΝΩΝΙΚΗ_ΠΡΟΝΟΙΑ 0.91 0.85 0.88 71 ΛΙΜΕΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.91 0.96 0.94 78 ΝΟΜΙΚΑ_ΠΡΟΣΩΠΑ_ΔΗΜΟΣΙΟΥ_ΔΙΚΑΙΟΥ 0.95 0.91 0.93 67 ΝΟΜΟΘΕΣΙΑ_ΑΝΩΝΥΜΩΝ_ΕΤΑΙΡΕΙΩΝ_ΤΡΑΠΕΖΩΝ_ΚΑΙ_ΧΡΗΜΑΤΙΣΤΗΡΙΩΝ 0.89 0.81 0.85 72 ΝΟΜΟΘΕΣΙΑ_ΔΗΜΩΝ_ΚΑΙ_ΚΟΙΝΟΤΗΤΩΝ 0.90 0.85 0.88 55 ΝΟΜΟΘΕΣΙΑ_ΕΠΙΜΕΛΗΤΗΡΙΩΝ_ΣΥΝΕΤΑΙΡΙΣΜΩΝ_ΚΑΙ_ΣΩΜΑΤΕΙΩΝ 0.92 0.92 0.92 37 ΟΙΚΟΝΟΜΙΚΗ_ΔΙΟΙΚΗΣΗ 0.86 0.70 0.77 53 ΠΕΡΙΟΥΣΙΑ_ΔΗΜΟΣΙΟΥ_ΚΑΙ_ΝΟΜΙΣΜΑ 0.80 0.80 0.80 46 ΠΟΙΝΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.96 0.94 0.95 49 ΠΟΛΕΜΙΚΗ_ΑΕΡΟΠΟΡΙΑ 0.89 0.83 0.86 29 ΠΟΛΕΜΙΚΟ_ΝΑΥΤΙΚΟ 0.94 0.77 0.85 43 ΠΟΛΙΤΙΚΗ_ΑΕΡΟΠΟΡΙΑ 0.93 0.90 0.92 30 ΠΟΛΙΤΙΚΗ_ΔΙΚΟΝΟΜΙΑ 0.86 0.95 0.90 19 ΡΑΔΙΟΦΩΝΙΑ_ΚΑΙ_ΤΥΠΟΣ 0.81 0.94 0.87 31 ΣΤΡΑΤΟΣ_ΞΗΡΑΣ 0.74 0.83 0.78 71 ΣΥΓΚΟΙΝΩΝΙΕΣ 0.91 0.92 0.91 178 ΣΥΝΤΑΓΜΑΤΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.84 0.74 0.79 87 ΤΑΧΥΔΡΟΜΕΙΑ_ΤΗΛΕΠΙΚΟΙΝΩΝΙΕΣ 0.96 0.93 0.95 84 ΤΕΛΩΝΕΙΑΚΗ_ΝΟΜΟΘΕΣΙΑ 0.87 0.84 0.85 73 ΤΥΠΟΣ_ΚΑΙ_ΤΟΥΡΙΣΜΟΣ 0.96 0.98 0.97 50 ΥΓΕΙΟΝΟΜΙΚΗ_ΝΟΜΟΘΕΣΙΑ 0.91 0.93 0.92 136 accuracy - - 0.88 3745 macro avg 0.87 0.87 0.87 3745 weighted avg 0.89 0.88 0.88 3745 ``` --- layout: model title: English RobertaForQuestionAnswering (from deepakvk) author: John Snow Labs name: roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2-finetuned-squad` is a English model originally trained by `deepakvk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735429401.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad_en_4.0.0_3.0_1655735429401.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base.by_deepakvk").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepakvk_roberta_base_squad2_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepakvk/roberta-base-squad2-finetuned-squad --- layout: model title: Sentence Detection in Indonesian Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [open_source, sentence_detection, id] task: Sentence Detection language: id edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_id_3.2.0_3.0_1630318954338.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_id_3.2.0_3.0_1630318954338.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "id") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""Mencari sumber paragraf bacaan bahasa Inggris yang bagus? Anda telah datang ke tempat yang tepat. Menurut sebuah penelitian baru-baru ini, kebiasaan membaca di kalangan remaja saat ini menurun dengan cepat. Mereka tidak dapat fokus pada paragraf bacaan bahasa Inggris yang diberikan selama lebih dari beberapa detik! Juga, membaca adalah dan merupakan bagian integral dari semua ujian kompetitif. Jadi, bagaimana Anda meningkatkan keterampilan membaca Anda? Jawaban atas pertanyaan ini sebenarnya adalah pertanyaan lain: Apa gunanya keterampilan membaca? Tujuan utama membaca adalah 'untuk masuk akal'.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "id") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("Mencari sumber paragraf bacaan bahasa Inggris yang bagus? Anda telah datang ke tempat yang tepat. Menurut sebuah penelitian baru-baru ini, kebiasaan membaca di kalangan remaja saat ini menurun dengan cepat. Mereka tidak dapat fokus pada paragraf bacaan bahasa Inggris yang diberikan selama lebih dari beberapa detik! Juga, membaca adalah dan merupakan bagian integral dari semua ujian kompetitif. Jadi, bagaimana Anda meningkatkan keterampilan membaca Anda? Jawaban atas pertanyaan ini sebenarnya adalah pertanyaan lain: Apa gunanya keterampilan membaca? Tujuan utama membaca adalah 'untuk masuk akal'.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python nlu.load('id.sentence_detector').predict("Mencari sumber paragraf bacaan bahasa Inggris yang bagus? Anda telah datang ke tempat yang tepat. Menurut sebuah penelitian baru-baru ini, kebiasaan membaca di kalangan remaja saat ini menurun dengan cepat. Mereka tidak dapat fokus pada paragraf bacaan bahasa Inggris yang diberikan selama lebih dari beberapa detik! Juga, membaca adalah dan merupakan bagian integral dari semua ujian kompetitif. Jadi, bagaimana Anda meningkatkan keterampilan membaca Anda? Jawaban atas pertanyaan ini sebenarnya adalah pertanyaan lain: Apa gunanya keterampilan membaca? Tujuan utama membaca adalah 'untuk masuk akal'.", output_level ='sentence') ```

## Results ```bash +---------------------------------------------------------------------------------------------------------------+ |result | +---------------------------------------------------------------------------------------------------------------+ |[Mencari sumber paragraf bacaan bahasa Inggris yang bagus?] | |[Anda telah datang ke tempat yang tepat.] | |[Menurut sebuah penelitian baru-baru ini, kebiasaan membaca di kalangan remaja saat ini menurun dengan cepat.] | |[Mereka tidak dapat fokus pada paragraf bacaan bahasa Inggris yang diberikan selama lebih dari beberapa detik!]| |[Juga, membaca adalah dan merupakan bagian integral dari semua ujian kompetitif.] | |[Jadi, bagaimana Anda meningkatkan keterampilan membaca Anda?] | |[Jawaban atas pertanyaan ini sebenarnya adalah pertanyaan lain:] | |[Apa gunanya keterampilan membaca?] | |[Tujuan utama membaca adalah 'untuk masuk akal'.] | +---------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|id| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Fast and Accurate Language Identification - 375 Languages (CNN) author: John Snow Labs name: ld_wiki_tatoeba_cnn_375 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The models are trained on large datasets such as Wikipedia and Tatoeba with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Abkhaz`, `Iraqi Arabic`, `Adyghe`, `Afrikaans`, `Gulf Arabic`, `Afrihili`, `Assyrian Neo-Aramaic`, `Ainu`, `Aklanon`, `Gheg Albanian`, `Amharic`, `Aragonese`, `Old English`, `Uab Meto`, `North Levantine Arabic`, `Arabic`, `Algerian Arabic`, `Moroccan Arabic`, `Egyptian Arabic`, `Assamese`, `Asturian`, `Kotava`, `Awadhi`, `Aymara`, `Azerbaijani`, `Bashkir`, `Baluchi`, `Balinese`, `Bavarian`, `Central Bikol`, `Belarusian`, `Berber`, `Bulgarian`, `Bhojpuri`, `Bislama`, `Banjar`, `Bambara`, `Bengali`, `Tibetan`, `Breton`, `Bodo`, `Bosnian`, `Buryat`, `Baybayanon`, `Brithenig`, `Catalan`, `Cayuga`, `Chavacano`, `Chechen`, `Cebuano`, `Chamorro`, `Chagatai`, `Chinook Jargon`, `Choctaw`, `Cherokee`, `Jin Chinese`, `Chukchi`, `Central Mnong`, `Corsican`, `Chinese Pidgin English`, `Crimean Tatar`, `Seychellois Creole`, `Czech`, `Kashubian`, `Chuvash`, `Welsh`, `CycL`, `Cuyonon`, `Danish`, `German`, `Dungan`, `Drents`, `Lower Sorbian`, `Central Dusun`, `Dhivehi`, `Dutton World Speedwords`, `Ewe`, `Emilian`, `Greek`, `Erromintxela`, `English`, `Middle English`, `Esperanto`, `Spanish`, `Estonian`, `Basque`, `Evenki`, `Extremaduran`, `Persian`, `Finnish`, `Fijian`, `Kven Finnish`, `Faroese`, `French`, `Middle French`, `Old French`, `North Frisian`, `Pulaar`, `Friulian`, `Nigerian Fulfulde`, `Frisian`, `Irish`, `Ga`, `Gagauz`, `Gan Chinese`, `Garhwali`, `Guadeloupean Creole French`, `Scottish Gaelic`, `Gilbertese`, `Galician`, `Guarani`, `Konkani (Goan)`, `Gronings`, `Gothic`, `Ancient Greek`, `Swiss German`, `Gujarati`, `Manx`, `Hausa`, `Hakka Chinese`, `Hawaiian`, `Ancient Hebrew`, `Hebrew`, `Hindi`, `Fiji Hindi`, `Hiligaynon`, `Hmong Njua (Green)`, `Ho`, `Croatian`, `Hunsrik`, `Upper Sorbian`, `Xiang Chinese`, `Haitian Creole`, `Hungarian`, `Armenian`, `Interlingua`, `Iban`, `Indonesian`, `Interlingue`, `Igbo`, `Nuosu`, `Inuktitut`, `Ilocano`, `Ido`, `Icelandic`, `Italian`, `Ingrian`, `Japanese`, `Jamaican Patois`, `Lojban`, `Juhuri (Judeo-Tat)`, `Jewish Palestinian Aramaic`, `Javanese`, `Georgian`, `Karakalpak`, `Kabyle`, `Kamba`, `Kekchi (Q'eqchi')`, `Khasi`, `Khakas`, `Kazakh`, `Greenlandic`, `Khmer`, `Kannada`, `Korean`, `Komi-Permyak`, `Komi-Zyrian`, `Karachay-Balkar`, `Karelian`, `Kashmiri`, `Kölsch`, `Kurdish`, `Kumyk`, `Cornish`, `Keningau Murut`, `Kyrgyz`, `Coastal Kadazan`, `Latin`, `Southern Subanen`, `Ladino`, `Luxembourgish`, `Láadan`, `Lingua Franca Nova`, `Luganda`, `Ligurian`, `Livonian`, `Lakota`, `Ladin`, `Lombard`, `Lingala`, `Lao`, `Louisiana Creole`, `Lithuanian`, `Latgalian`, `Latvian`, `Latvian`, `Literary Chinese`, `Laz`, `Madurese`, `Maithili`, `North Moluccan Malay`, `Moksha`, `Morisyen`, `Malagasy`, `Mambae`, `Marshallese`, `Meadow Mari`, `Maori`, `Mi'kmaq`, `Minangkabau`, `Macedonian`, `Malayalam`, `Mongolian`, `Manchu`, `Mon`, `Mohawk`, `Marathi`, `Hill Mari`, `Malay`, `Maltese`, `Tagal Murut`, `Mirandese`, `Hmong Daw (White)`, `Burmese`, `Erzya`, `Nauruan`, `Nahuatl`, `Norwegian Bokmål`, `Central Huasteca Nahuatl`, `Low German (Low Saxon)`, `Nepali`, `Newari`, `Ngeq`, `Guerrero Nahuatl`, `Niuean`, `Dutch`, `Orizaba Nahuatl`, `Norwegian Nynorsk`, `Norwegian`, `Nogai`, `Old Norse`, `Novial`, `Nepali`, `Naga (Tangshang)`, `Navajo`, `Chinyanja`, `Nyungar`, `Old Aramaic`, `Occitan`, `Ojibwe`, `Odia (Oriya)`, `Old East Slavic`, `Ossetian`, `Old Spanish`, `Old Saxon`, `Ottoman Turkish`, `Old Turkish`, `Punjabi (Eastern)`, `Pangasinan`, `Kapampangan`, `Papiamento`, `Palauan`, `Picard`, `Pennsylvania German`, `Palatine German`, `Phoenician`, `Pali`, `Polish`, `Piedmontese`, `Punjabi (Western)`, `Pipil`, `Old Prussian`, `Pashto`, `Portuguese`, `Quechua`, `K'iche'`, `Quenya`, `Rapa Nui`, `Rendille`, `Tarifit`, `Romansh`, `Kirundi`, `Romanian`, `Romani`, `Russian`, `Rusyn`, `Kinyarwanda`, `Okinawan`, `Sanskrit`, `Yakut`, `Sardinian`, `Sicilian`, `Scots`, `Sindhi`, `Northern Sami`, `Sango`, `Samogitian`, `Shuswap`, `Tachawit`, `Sinhala`, `Sindarin`, `Slovak`, `Slovenian`, `Samoan`, `Southern Sami`, `Shona`, `Somali`, `Albanian`, `Serbian`, `Swazi`, `Southern Sotho`, `Saterland Frisian`, `Sundanese`, `Sumerian`, `Swedish`, `Swahili`, `Swabian`, `Swahili`, `Syriac`, `Tamil`, `Telugu`, `Tetun`, `Tajik`, `Thai`, `Tahaggart Tamahaq`, `Tigrinya`, `Tigre`, `Turkmen`, `Tokelauan`, `Tagalog`, `Klingon`, `Talysh`, `Jewish Babylonian Aramaic`, `Temuan`, `Setswana`, `Tongan`, `Tonga (Zambezi)`, `Toki Pona`, `Tok Pisin`, `Old Tupi`, `Turkish`, `Tsonga`, `Tatar`, `Isan`, `Tuvaluan`, `Tahitian`, `Tuvinian`, `Talossan`, `Udmurt`, `Uyghur`, `Ukrainian`, `Umbundu`, `Urdu`, `Urhobo`, `Uzbek`, `Venetian`, `Veps`, `Vietnamese`, `Volapük`, `Võro`, `Walloon`, `Waray`, `Wolof`, `Shanghainese`, `Kalmyk`, `Xhosa`, `Mingrelian`, `Yiddish`, `Yoruba`, `Cantonese`, `Chinese`, `Malay (Vernacular)`, `Malay`, `Zulu`, `Zaza`. ## Predicted Entities `ab`, `acm`, `ady`, `af`, `afb`, `afh`, `aii`, `ain`, `akl`, `aln`, `am`, `an`, `ang`, `aoz`, `apc`, `ar`, `arq`, `ary`, `arz`, `as`, `ast`, `avk`, `awa`, `ay`, `az`, `ba`, `bal`, `ban`, `bar`, `bcl`, `be`, `ber`, `bg`, `bho`, `bi`, `bjn`, `bm`, `bn`, `bo`, `br`, `brx`, `bs`, `bua`, `bvy`, `bzt`, `ca`, `cay`, `cbk`, `ce`, `ceb`, `ch`, `chg`, `chn`, `cho`, `chr`, `cjy`, `ckt`, `cmo`, `co`, `cpi`, `crh`, `crs`, `cs`, `csb`, `cv`, `cy`, `cycl`, `cyo`, `da`, `de`, `dng`, `drt`, `dsb`, `dtp`, `dv`, `dws`, `ee`, `egl`, `el`, `emx`, `en`, `enm`, `eo`, `es`, `et`, `eu`, `evn`, `ext`, `fa`, `fi`, `fj`, `fkv`, `fo`, `fr`, `frm`, `fro`, `frr`, `fuc`, `fur`, `fuv`, `fy`, `ga`, `gaa`, `gag`, `gan`, `gbm`, `gcf`, `gd`, `gil`, `gl`, `gn`, `gom`, `gos`, `got`, `grc`, `gsw`, `gu`, `gv`, `ha`, `hak`, `haw`, `hbo`, `he`, `hi`, `hif`, `hil`, `hnj`, `hoc`, `hr`, `hrx`, `hsb`, `hsn`, `ht`, `hu`, `hy`, `ia`, `iba`, `id`, `ie`, `ig`, `ii`, `ike`, `ilo`, `io`, `is`, `it`, `izh`, `ja`, `jam`, `jbo`, `jdt`, `jpa`, `jv`, `ka`, `kaa`, `kab`, `kam`, `kek`, `kha`, `kjh`, `kk`, `kl`, `km`, `kn`, `ko`, `koi`, `kpv`, `krc`, `krl`, `ks`, `ksh`, `ku`, `kum`, `kw`, `kxi`, `ky`, `kzj`, `la`, `laa`, `lad`, `lb`, `ldn`, `lfn`, `lg`, `lij`, `liv`, `lkt`, `lld`, `lmo`, `ln`, `lo`, `lou`, `lt`, `ltg`, `lv`, `lvs`, `lzh`, `lzz`, `mad`, `mai`, `max`, `mdf`, `mfe`, `mg`, `mgm`, `mh`, `mhr`, `mi`, `mic`, `min`, `mk`, `ml`, `mn`, `mnc`, `mnw`, `moh`, `mr`, `mrj`, `ms`, `mt`, `mvv`, `mwl`, `mww`, `my`, `myv`, `na`, `nah`, `nb`, `nch`, `nds`, `ne`, `new`, `ngt`, `ngu`, `niu`, `nl`, `nlv`, `nn`, `no`, `nog`, `non`, `nov`, `npi`, `nst`, `nv`, `ny`, `nys`, `oar`, `oc`, `oj`, `or`, `orv`, `os`, `osp`, `osx`, `ota`, `otk`, `pa`, `pag`, `pam`, `pap`, `pau`, `pcd`, `pdc`, `pfl`, `phn`, `pi`, `pl`, `pms`, `pnb`, `ppl`, `prg`, `ps`, `pt`, `qu`, `quc`, `qya`, `rap`, `rel`, `rif`, `rm`, `rn`, `ro`, `rom`, `ru`, `rue`, `rw`, `ryu`, `sa`, `sah`, `sc`, `scn`, `sco`, `sd`, `se`, `sg`, `sgs`, `shs`, `shy`, `si`, `sjn`, `sk`, `sl`, `sm`, `sma`, `sn`, `so`, `sq`, `sr`, `ss`, `st`, `stq`, `su`, `sux`, `sv`, `sw`, `swg`, `swh`, `syc`, `ta`, `te`, `tet`, `tg`, `th`, `thv`, `ti`, `tig`, `tk`, `tkl`, `tl`, `tlh`, `tly`, `tmr`, `tmw`, `tn`, `to`, `toi`, `toki`, `tpi`, `tpw`, `tr`, `ts`, `tt`, `tts`, `tvl`, `ty`, `tyv`, `tzl`, `udm`, `ug`, `uk`, `umb`, `ur`, `urh`, `uz`, `vec`, `vep`, `vi`, `vo`, `vro`, `wa`, `war`, `wo`, `wuu`, `xal`, `xh`, `xmf`, `yi`, `yo`, `yue`, `zh`, `zlm`, `zsm`, `zu`, `zza`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_375_xx_2.7.0_2.4_1607184873730.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_wiki_tatoeba_cnn_375_xx_2.7.0_2.4_1607184873730.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_375", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_wiki_tatoeba_cnn_375", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_375').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_wiki_tatoeba_cnn_375| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Wikipedia and Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | fr| 1000| 1000| 1.0| | de| 1000| 999| 0.999| | fi| 1000| 999| 0.999| | nl| 1000| 998| 0.998| | el| 1000| 997| 0.997| | en| 1000| 995| 0.995| | es| 1000| 994| 0.994| | it| 1000| 993| 0.993| | sv| 1000| 991| 0.991| | da| 1000| 987| 0.987| | pl| 914| 901|0.9857768052516411| | hu| 880| 866|0.9840909090909091| | pt| 1000| 980| 0.98| | et| 928| 907|0.9773706896551724| | ro| 784| 766|0.9770408163265306| | lt| 1000| 976| 0.976| | bg| 1000| 965| 0.965| | cs| 1000| 945| 0.945| | sk| 1000| 944| 0.944| | lv| 916| 843|0.9203056768558951| | sl| 914| 810|0.8862144420131292| +--------+-----+-------+------------------+ +-------+--------------------+ |summary| precision| +-------+--------------------+ | count| 21| | mean| 0.9758952066282511| | stddev|0.029434744995013935| | min| 0.8862144420131292| | max| 1.0| +-------+--------------------+ ``` --- layout: model title: English asr_wav2vec2_base_checkpoint_9 TFWav2Vec2ForCTC from jiobiala24 author: John Snow Labs name: asr_wav2vec2_base_checkpoint_9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_checkpoint_9` is a English model originally trained by jiobiala24. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_checkpoint_9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_checkpoint_9_en_4.2.0_3.0_1664021017779.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_checkpoint_9_en_4.2.0_3.0_1664021017779.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_checkpoint_9", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_checkpoint_9", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_checkpoint_9| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|349.2 MB| --- layout: model title: Word2Vec Embeddings in Portuguese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, pt, open_source] task: Embeddings language: pt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pt_3.4.1_3.0_1647453502800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pt_3.4.1_3.0_1647453502800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_skin_type", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_skin_type", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_skin_type| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Extract Cancer Therapies and Posology Information author: John Snow Labs name: ner_oncology_unspecific_posology date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner, treatment, posology] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of treatments and posology information using unspecific labels (low granularity). Definitions of Predicted Entities: - `Cancer_Therapy`: Mentions of cancer treatments, including chemotherapy, radiotherapy, surgery and other. - `Posology_Information`: Terms related to the posology of the treatment, including duration, frequencies and dosage. ## Predicted Entities `Cancer_Therapy`, `Posology_Information` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_en_4.0.0_3.0_1666722206468.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_en_4.0.0_3.0_1666722206468.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------|:---------------------| | adriamycin | Cancer_Therapy | | 60 mg/m2 | Posology_Information | | cyclophosphamide | Cancer_Therapy | | 600 mg/m2 | Posology_Information | | over six courses | Posology_Information | | second cycle | Posology_Information | | chemotherapy | Cancer_Therapy | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_unspecific_posology| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.3 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Posology_Information 2663 244 399 3062 0.92 0.87 0.89 Cancer_Therapy 2580 317 247 2827 0.89 0.91 0.90 macro_avg 5243 561 646 5889 0.90 0.89 0.90 micro_avg 5243 561 646 5889 0.90 0.89 0.90 ``` --- layout: model title: English asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm TFWav2Vec2ForCTC from gxbag author: John Snow Labs name: pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm` is a English model originally trained by gxbag. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm_en_4.2.0_3.0_1664026375800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm_en_4.2.0_3.0_1664026375800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_960h_lv60_self_with_wikipedia_lm| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|333.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Japanese Bert Embeddings (Base, v2) author: John Snow Labs name: bert_embeddings_bert_base_japanese_v2 date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-v2` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_v2_ja_3.4.2_3.0_1649674281970.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_v2_ja_3.4.2_3.0_1649674281970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_v2| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|417.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-v2 - https://github.com/google-research/bert - https://pypi.org/project/unidic-lite/ - https://github.com/cl-tohoku/bert-japanese/tree/v2.0 - https://taku910.github.io/mecab/ - https://github.com/neologd/mecab-ipadic-neologd - https://github.com/polm/fugashi - https://github.com/polm/unidic-lite - https://www.tensorflow.org/tfrc/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: Sentiment Analysis Pipeline for French texts author: John Snow Labs name: classifierdl_bert_sentiment_pipeline date: 2021-09-28 tags: [fr, sentiment, pipeline, open_source] task: Sentiment Analysis language: fr edition: Spark NLP 3.3.0 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline identifies the sentiments (positive or negative) in French texts. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/SENTIMENT_FR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_Fr_Sentiment.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_bert_sentiment_pipeline_fr_3.3.0_2.4_1632835775093.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_bert_sentiment_pipeline_fr_3.3.0_2.4_1632835775093.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_bert_sentiment_pipeline", lang = "fr") result = pipeline.annotate("Mignolet vraiment dommage de ne jamais le voir comme titulaire") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_bert_sentiment_pipeline", lang = "fr") val result = pipeline.fullAnnotate("Mignolet vraiment dommage de ne jamais le voir comme titulaire")(0) ```

## Results ```bash ['NEGATIVE'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_sentiment_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.0+| |License:|Open Source| |Edition:|Official| |Language:|fr| ## Included Models - DocumentAssembler - BertSentenceEmbeddings - ClassifierDLModel --- layout: model title: English DistilBertForTokenClassification Cased model (from f2io) author: John Snow Labs name: distilbert_token_classifier_ner_roles_openapi date: 2023-03-03 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ner-roles-openapi` is a English model originally trained by `f2io`. ## Predicted Entities ``, `LOC`, `OR`, `PRG`, `ROLE`, `ORG`, `PER`, `ENTITY`, `MISC`, `ACTION` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.0_3.0_1677881330949.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_ner_roles_openapi_en_4.3.0_3.0_1677881330949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_ner_roles_openapi","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_ner_roles_openapi| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/f2io/ner-roles-openapi --- layout: model title: Word2Vec Embeddings in Tajik (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, tg, open_source] task: Embeddings language: tg edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tg_3.4.1_3.0_1647461560049.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_tg_3.4.1_3.0_1647461560049.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_bert_news_pipeline", lang = "ur") result = pipeline.fullAnnotate("""گزشتہ ہفتے ایپل کے حصص میں 11 فیصد اضافہ ہوا ہے۔""") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_bert_news_pipeline", "ur") val result = pipeline.fullAnnotate("گزشتہ ہفتے ایپل کے حصص میں 11 فیصد اضافہ ہوا ہے۔")(0) ```

## Results ```bash ['business'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_news_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Language:|ur| |Size:|1.8 GB| ## Included Models - DocumentAssembler - BertSentenceEmbeddings - ClassifierDLModel --- layout: model title: English image_classifier_vit_planes_trains_automobiles ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_planes_trains_automobiles date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_planes_trains_automobiles` is a English model originally trained by nateraw. ## Predicted Entities `automobiles`, `planes`, `trains` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_planes_trains_automobiles_en_4.1.0_3.0_1660170662986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_planes_trains_automobiles_en_4.1.0_3.0_1660170662986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_planes_trains_automobiles", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_planes_trains_automobiles", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_planes_trains_automobiles| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: RCT Binary Classifier (USE) author: John Snow Labs name: rct_binary_classifier_use date: 2022-05-27 tags: [licensed, clinical, rct, classifier, en] task: Text Classification language: en nav_key: models edition: Healthcare NLP 3.4.2 spark_version: 3.0 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a USE based classifier that can classify if an article is a randomized clinical trial (RCT) or not. ## Predicted Entities `true`, `false` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_RCT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CLASSIFICATION_RCT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rct_binary_classifier_use_en_3.4.2_3.0_1653676810143.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rct_binary_classifier_use_en_3.4.2_3.0_1653676810143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") classifier_dl = ClassifierDLModel.pretrained("rct_binary_classifier_use", "en", "clinical/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("class") use_clf_pipeline = Pipeline( stages = [ document_assembler, use, classifier_dl ]) data = spark.createDataFrame([["""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """]]).toDF("text") result = use_clf_pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val use = UniversalSentenceEncoder.pretrained() .setInputCols("document") .setOutputCol("sentence_embeddings") val classifier_dl = ClassifierDLModel.pretrained("rct_binary_classifier_use", "en", "clinical/models") .setInputCols(Array("sentence_embeddings")) .setOutputCol("class") val use_clf_pipeline = new Pipeline().setStages(Array(documenter, use, classifier_dl)) val data = Seq("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """).toDS.toDF("text") val result = use_clf_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.rct_binary_use").predict("""Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. """) ```

## Results ```bash | text | rct | |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------| | Abstract:Based on the American Society of Anesthesiologists' Practice Guidelines for Sedation and Analgesia by Non-Anesthesiologists (ASA-SED), a sedation training course aimed at improving medical safety was developed by the Japanese Association for Medical Simulation in 2011. This study evaluated the effect of debriefing on participants' perceptions of the essential points of the ASA-SED. A total of 38 novice doctors participated in the sedation training course during the research period. Of these doctors, 18 participated in the debriefing group, and 20 participated in non-debriefing group. Scoring of participants' guideline perceptions was conducted using an evaluation sheet (nine items, 16 points) created based on the ASA-SED. The debriefing group showed a greater perception of the ASA-SED, as reflected in the significantly higher scores on the evaluation sheet (median, 16 points) than the control group (median, 13 points; p < 0.05). No significant differences were identified before or during sedation, but the difference after sedation was significant (p < 0.05). Debriefing after sedation training courses may contribute to better perception of the ASA-SED, and may lead to enhanced attitudes toward medical safety during sedation and analgesia. | true | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rct_binary_classifier_use| |Compatibility:|Healthcare NLP 3.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|20.9 MB| ## References https://arxiv.org/abs/1710.06071 ## Benchmarking ```bash label precision recall f1-score support false 0.84 0.80 0.82 2915 true 0.78 0.82 0.80 2545 accuracy - - 0.81 5460 macro-avg 0.81 0.81 0.81 5460 weighted-avg 0.81 0.81 0.81 5460 ``` --- layout: model title: Spanish RobertaForQuestionAnswering (from nlp-en-es) author: John Snow Labs name: roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-finetuned-sqac` is a Spanish model originally trained by `nlp-en-es`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac_es_4.0.0_3.0_1655730047157.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac_es_4.0.0_3.0_1655730047157.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.sqac.roberta.base.by_nlp-en-es").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_nlp_en_es_roberta_base_bne_finetuned_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|460.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlp-en-es/roberta-base-bne-finetuned-sqac - https://paperswithcode.com/sota?task=Question+Answering&dataset=sqac --- layout: model title: Word2Vec Embeddings in Mazanderani (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, mzn, open_source] task: Embeddings language: mzn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_mzn_3.4.1_3.0_1647445915568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_mzn_3.4.1_3.0_1647445915568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_fa_base_uncased_ner_arman| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|607.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/HooshvareLab/bert-fa-base-uncased-ner-arman - https://github.com/hooshvare/parsbert - https://github.com/HaniehP/PersianNER - https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb - https://github.com/hooshvare/parsbert/issues --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223261564.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0_en_4.3.0_3.0_1674223261564.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_bert_quadruplet_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_bert_quadruplet_epochs_1_shard_1_squad2.0 --- layout: model title: English RobertaForQuestionAnswering Mini Cased model (from sguskin) author: John Snow Labs name: roberta_qa_dynamic_minilmv2_l6_h384_squad1.1 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dynamic-minilmv2-L6-H384-squad1.1` is a English model originally trained by `sguskin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_dynamic_minilmv2_l6_h384_squad1.1_en_4.3.0_3.0_1674210790213.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_dynamic_minilmv2_l6_h384_squad1.1_en_4.3.0_3.0_1674210790213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_dynamic_minilmv2_l6_h384_squad1.1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_dynamic_minilmv2_l6_h384_squad1.1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_dynamic_minilmv2_l6_h384_squad1.1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|112.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/sguskin/dynamic-minilmv2-L6-H384-squad1.1 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from rbiswas4) author: John Snow Labs name: distilbert_qa_rbiswas4_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `rbiswas4`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_rbiswas4_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772274920.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_rbiswas4_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772274920.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rbiswas4_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_rbiswas4_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_rbiswas4_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/rbiswas4/distilbert-base-uncased-finetuned-squad --- layout: model title: Named Entity Recognition for Korean (GloVe 840B 300d) author: John Snow Labs name: ner_kmou_glove_840B_300d date: 2021-01-03 task: Named Entity Recognition language: ko edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ko, ner, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates named entities in a text, that can be used to find features such as names of people, places, and organizations in the `BIO` format. The model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. This model uses the pre-trained `glove_840B_300` embeddings model from `WordEmbeddings` annotator as an input, so be sure to use the same embeddings in the pipeline. ## Predicted Entities Dates-`DT`, Locations-`LC`, Organizations-`OG`, Persons-`PS`, Time-`TI`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_KO/){:.button.button-orange.button-orange-trans.co.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ner_kmou_glove_840B_300d_ko_2.7.0_2.4_1609716021199.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ner_kmou_glove_840B_300d_ko_2.7.0_2.4_1609716021199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_kaist_ud", "ko")\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", "xx")\ .setInputCols("document", "token") \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("ner_kmou_glove_840B_300d", "ko") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter]) example = spark.createDataFrame([['라이프니츠 의 주도 로 베를린 에 세우 어 지 ㄴ 베를린 과학아카데미']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_kaist_ud", "ko") .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", "xx") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_kmou_glove_840B_300d", "ko") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter)) val data = Seq("라이프니츠 의 주도 로 베를린 에 세우 어 지 ㄴ 베를린 과학아카데미").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["라이프니츠 의 주도 로 베를린 에 세우 어 지 ㄴ 베를린 과학아카데미"] ner_df = nlu.load('ko.ner').predict(text) ner_df ```

## Results ```bash +------------+----+ |token |ner | +------------+----+ |라이프니츠 |B-PS| |베를린 |B-OG| |과학아카데미 |I-OG| +------------+----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_kmou_glove_840B_300d| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ko| ## Data Source The model was trained by the Korea Maritime and Ocean University [NLP data set](https://github.com/kmounlp/NER). ## Benchmarking ```bash | ner_tag | precision | recall | f1-score | support | |:------------:|:---------:|:------:|:--------:|:-------:| | B-DT | 0.95 | 0.29 | 0.44 | 132 | | B-LC | 0.00 | 0.00 | 0.00 | 166 | | B-OG | 1.00 | 0.06 | 0.11 | 149 | | B-PS | 0.86 | 0.13 | 0.23 | 287 | | B-TI | 0.50 | 0.05 | 0.09 | 20 | | I-DT | 0.94 | 0.36 | 0.52 | 164 | | I-LC | 0.00 | 0.00 | 0.00 | 4 | | I-OG | 1.00 | 0.08 | 0.15 | 25 | | I-PS | 1.00 | 0.08 | 0.15 | 12 | | I-TI | 0.50 | 0.10 | 0.17 | 10 | | O | 0.94 | 1.00 | 0.97 | 12830 | | accuracy | 0.94 | 13799 | | | | macro avg | 0.70 | 0.20 | 0.26 | 13799 | | weighted avg | 0.93 | 0.94 | 0.92 | 13799 | ``` --- layout: model title: English image_classifier_vit_Check_Gum_Teeth ViTForImageClassification from steven123 author: John Snow Labs name: image_classifier_vit_Check_Gum_Teeth date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Check_Gum_Teeth` is a English model originally trained by steven123. ## Predicted Entities `Bad_Gum`, `Good_Gum` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Check_Gum_Teeth_en_4.1.0_3.0_1660168122164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Check_Gum_Teeth_en_4.1.0_3.0_1660168122164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Check_Gum_Teeth", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Check_Gum_Teeth", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Check_Gum_Teeth| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squadv1-x1.96-f88.3-d27-hybrid-filled-opt-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1_en_4.0.0_3.0_1654181570985.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1_en_4.0.0_3.0_1654181570985.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_x1.84_f88.7_d36_hybrid_filled_v1.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squadv1_x1.96_f88.3_d27_hybrid_filled_opt_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|188.1 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squadv1-x1.96-f88.3-d27-hybrid-filled-opt-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: English asr_wav2vec2_xls_r_1b_english TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_wav2vec2_xls_r_1b_english date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_1b_english` is a English model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_1b_english_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_1b_english_en_4.2.0_3.0_1664015553321.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_1b_english_en_4.2.0_3.0_1664015553321.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_1b_english", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_1b_english", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_1b_english| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|3.6 GB| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_terri1102 TFWav2Vec2ForCTC from terri1102 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_terri1102` is a English model originally trained by terri1102. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102_en_4.2.0_3.0_1664107472195.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102_en_4.2.0_3.0_1664107472195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_terri1102| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: German DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_de_cased date: 2022-04-12 tags: [distilbert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-de-cased` is a German model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_de_cased_de_3.4.2_3.0_1649783710575.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_de_cased_de_3.4.2_3.0_1649783710575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_de_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|236.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-de-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Bemba (Zambia) asr_wav2vec2_large_xlsr_bemba TFWav2Vec2ForCTC from csikasote author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_bemba date: 2022-09-24 tags: [wav2vec2, bem, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: bem edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_bemba` is a Bemba (Zambia) model originally trained by csikasote. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_bemba_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_bemba_bem_4.2.0_3.0_1664023162676.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_bemba_bem_4.2.0_3.0_1664023162676.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_bemba', lang = 'bem') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_bemba", lang = "bem") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_bemba| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|bem| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering Cased model (from yirenl2) author: John Snow Labs name: roberta_qa_plm date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `plm_qa` is a English model originally trained by `yirenl2`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_plm_en_4.2.4_3.0_1669985368479.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_plm_en_4.2.4_3.0_1669985368479.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_plm","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_plm","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_plm| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/yirenl2/plm_qa - https://paperswithcode.com/sota?task=Question+Answering&dataset=squad_v2 --- layout: model title: English image_classifier_vit_rare_puppers2 ViTForImageClassification from Samlit author: John Snow Labs name: image_classifier_vit_rare_puppers2 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rare_puppers2` is a English model originally trained by Samlit. ## Predicted Entities ` La Goulue Toulouse-Lautrec`, `aristide bruant Lautrec`, `la goulue Toulouse-Lautrec`, `Marcelle Lender Bolero`, `Salon at the Rue des Moulins Lautrec` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_puppers2_en_4.1.0_3.0_1660171547389.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rare_puppers2_en_4.1.0_3.0_1660171547389.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rare_puppers2", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rare_puppers2", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rare_puppers2| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Word Segmenter for Japanese author: John Snow Labs name: wordseg_gsd_ud date: 2021-03-09 tags: [word_segmentation, open_source, japanese, wordseg_gsd_ud, ja] task: Word Segmentation language: ja edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: WordSegmenterModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [WordSegmenterModel-WSM](https://en.wikipedia.org/wiki/Text_segmentation) is based on maximum entropy probability model to detect word boundaries in Japanese text. Japanese text is written without white space between the words, and a computer-based application cannot know a priori which sequence of ideograms form a word. In many natural language processing tasks such as part-of-speech (POS) and named entity recognition (NER) require word segmentation as a initial step. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/chinese/word_segmentation/words_segmenter_demo.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_ja_3.0.0_3.0_1615292309908.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wordseg_gsd_ud_ja_3.0.0_3.0_1615292309908.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") .setInputCols(["sentence"]) .setOutputCol("token") pipeline = Pipeline(stages=[document_assembler, word_segmenter]) ws_model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) example = spark.createDataFrame([['ジョンスノーラボからこんにちは！ ']], ["text"]) result = ws_model.transform(example) ``` ```scala val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") .setInputCols(Array("sentence")) .setOutputCol("token") val pipeline = new Pipeline().setStages(Array(document_assembler, word_segmenter)) val data = Seq("ジョンスノーラボからこんにちは！ ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""ジョンスノーラボからこんにちは！ ""] token_df = nlu.load('ja.segment_words').predict(text) token_df ```

## Results ```bash 0 ジョンス 1 ノ 2 ー 3 ラ 4 ボ 5 から 6 こん 7 に 8 ち 9 は 10 ！ Name: token, dtype: object ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wordseg_gsd_ud| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[words_segmented]| |Language:|ja| --- layout: model title: English DistilBertForQuestionAnswering Cased model (from SauravMaheshkar) author: John Snow Labs name: distilbert_qa_base_cased_distilled_chaii date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_chaii_en_4.0.0_3.0_1654723547031.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_distilled_chaii_en_4.0.0_3.0_1654723547031.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_distilled_chaii","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.distil_bert.base_cased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_distilled_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/distilbert-base-cased-distilled-chaii --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_3_lr_2e_5_bs_32_ep_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-set_date_3-lr-2e-5-bs-32-ep-4` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_3_lr_2e_5_bs_32_ep_4_en_4.0.0_3.0_1657188613886.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_3_lr_2e_5_bs_32_ep_4_en_4.0.0_3.0_1657188613886.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_3_lr_2e_5_bs_32_ep_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_3_lr_2e_5_bs_32_ep_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_3_lr_2e_5_bs_32_ep_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-set_date_3-lr-2e-5-bs-32-ep-4 --- layout: model title: Embeddings Scielowiki 300 dims author: John Snow Labs name: embeddings_scielowiki_300d class: WordEmbeddingsModel language: es repository: clinical/models date: 2020-05-26 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,es] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_scielowiki_300d_es_2.5.0_2.4_1590467643391.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_scielowiki_300d_es_2.5.0_2.4_1590467643391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models") .setInputCols(["document","token"]) .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_scielowiki_300d","es","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("es.embed.scielowiki.300d").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_scielowiki_300d``. {:.model-param} ## Model Information {:.table-model} |---------------|----------------------------| | Name: | embeddings_scielowiki_300d | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: |[word_embeddings] | | Language: | es | | Dimension: | 300.0 | {:.h2_title} ## Data Source Trained on Scielo Articles + Clinical Wikipedia Articles https://zenodo.org/record/3744326#.XtViinVKh_U --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8_en_4.0.0_3.0_1657192728608.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8_en_4.0.0_3.0_1657192728608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|378.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-8 --- layout: model title: Topic Identification (Banking) author: John Snow Labs name: finclf_bert_banking77 date: 2022-09-28 tags: [en, finance, bank, topic, classification, modeling, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Bert-based model, which can be used to classify texts into 77 banking-related classes. This is a Multiclass model, meaning only one label will be returned as an output. The classes are the following: - activate_my_card - age_limit - apple_pay_or_google_pay - atm_support - automatic_top_up - balance_not_updated_after_bank_transfer - balance_not_updated_after_cheque_or_cash_deposit - beneficiary_not_allowed - cancel_transfer - card_about_to_expire - card_acceptance - card_arrival - card_delivery_estimate - card_linking - card_not_working - card_payment_fee_charged - card_payment_not_recognised - card_payment_wrong_exchange_rate - card_swallowed - cash_withdrawal_charge - cash_withdrawal_not_recognised - change_pin - compromised_card - contactless_not_working - country_support - declined_card_payment - declined_cash_withdrawal - declined_transfer - direct_debit_payment_not_recognised - disposable_card_limits - edit_personal_details - exchange_charge - exchange_rate - exchange_via_app - extra_charge_on_statement - failed_transfer - fiat_currency_support - get_disposable_virtual_card - get_physical_card - getting_spare_card - getting_virtual_card - lost_or_stolen_card - lost_or_stolen_phone - order_physical_card - passcode_forgotten - pending_card_payment - pending_cash_withdrawal - pending_top_up - pending_transfer - pin_blocked - receiving_money - Refund_not_showing_up - request_refund - reverted_card_payment? - supported_cards_and_currencies - terminate_account - top_up_by_bank_transfer_charge - top_up_by_card_charge - top_up_by_cash_or_cheque - top_up_failed - top_up_limits - top_up_reverted - topping_up_by_card - transaction_charged_twice - transfer_fee_charged - transfer_into_account - transfer_not_received_by_recipient - transfer_timing - unable_to_verify_identity - verify_my_identity - verify_source_of_funds - verify_top_up - virtual_card_not_working - visa_or_mastercard - why_verify_identity - wrong_amount_of_cash_received - wrong_exchange_rate_for_cash_withdrawal ## Predicted Entities `activate_my_card`, `age_limit`, `card_acceptance`, `card_arrival`, `card_delivery_estimate`, `card_linking`, `card_not_working`, `card_payment_fee_charged`, `card_payment_not_recognised`, `card_payment_wrong_exchange_rate`, `card_swallowed`, `cash_withdrawal_charge`, `apple_pay_or_google_pay`, `cash_withdrawal_not_recognised`, `change_pin`, `compromised_card`, `contactless_not_working`, `country_support`, `declined_card_payment`, `declined_cash_withdrawal`, `declined_transfer`, `direct_debit_payment_not_recognised`, `disposable_card_limits`, `atm_support`, `edit_personal_details`, `exchange_charge`, `exchange_rate`, `exchange_via_app`, `extra_charge_on_statement`, `failed_transfer`, `fiat_currency_support`, `get_disposable_virtual_card`, `get_physical_card`, `getting_spare_card`, `automatic_top_up`, `getting_virtual_card`, `lost_or_stolen_card`, `lost_or_stolen_phone`, `order_physical_card`, `passcode_forgotten`, `pending_card_payment`, `pending_cash_withdrawal`, `pending_top_up`, `pending_transfer`, `pin_blocked`, `balance_not_updated_after_bank_transfer`, `receiving_money`, `Refund_not_showing_up`, `request_refund`, `reverted_card_payment?`, `supported_cards_and_currencies`, `terminate_account`, `top_up_by_bank_transfer_charge`, `top_up_by_card_charge`, `top_up_by_cash_or_cheque`, `top_up_failed`, `balance_not_updated_after_cheque_or_cash_deposit`, `top_up_limits`, `top_up_reverted`, `topping_up_by_card`, `transaction_charged_twice`, `transfer_fee_charged`, `transfer_into_account`, `transfer_not_received_by_recipient`, `transfer_timing`, `unable_to_verify_identity`, `verify_my_identity`, `beneficiary_not_allowed`, `verify_source_of_funds`, `verify_top_up`, `virtual_card_not_working`, `visa_or_mastercard`, `why_verify_identity`, `wrong_amount_of_cash_received`, `wrong_exchange_rate_for_cash_withdrawal`, `cancel_transfer`, `card_about_to_expire` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_BANKING/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_bert_banking77_en_1.0.0_3.0_1664361071567.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_bert_banking77_en_1.0.0_3.0_1664361071567.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['atm_support'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_bert_banking77| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|410.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References Banking77 dataset (https://paperswithcode.com/dataset/banking77-oos) and in-house data augmentation ## Benchmarking ```bash label Score Loss 0.3031957447528839 Accuracy 0.9363636363636364 Macro_F1 0.9364655956915154 Micro_F1 0.9363636363636364 Weighted_F1 0.9364655956915157 Macro_Precision 0.9396792003322154 Micro_Precision 0.9363636363636364 Weighted_Precision 0.9396792003322155 Macro_Recall 0.9363636363636365 Micro_Recall 0.9363636363636364 Weighted_Recall 0.9363636363636364 ``` --- layout: model title: English ElectraForQuestionAnswering Large model (from sultan) author: John Snow Labs name: electra_qa_BioM_Large_SQuAD2 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BioM-ELECTRA-Large-SQuAD2` is a English model originally trained by `sultan`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Large_SQuAD2_en_4.0.0_3.0_1655919022213.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_BioM_Large_SQuAD2_en_4.0.0_3.0_1655919022213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_BioM_Large_SQuAD2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_BioM_Large_SQuAD2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.electra.large").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_BioM_Large_SQuAD2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/sultan/BioM-ELECTRA-Large-SQuAD2 - http://participants-area.bioasq.org/results/9b/phaseB/ - https://github.com/salrowili/BioM-Transformers --- layout: model title: Indonesian RobertaForMaskedLM Base Cased model (from cahya) author: John Snow Labs name: roberta_embeddings_base_indonesian_522m date: 2022-12-12 tags: [id, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: id edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-indonesian-522M` is a Indonesian model originally trained by `cahya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_base_indonesian_522m_id_4.2.4_3.0_1670859219449.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_base_indonesian_522m_id_4.2.4_3.0_1670859219449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_base_indonesian_522m","id") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_base_indonesian_522m","id") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_base_indonesian_522m| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|id| |Size:|473.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/cahya/roberta-base-indonesian-522M - https://github.com/cahya-wirawan/indonesian-language-models/tree/master/Transformers --- layout: model title: Word2Vec Embeddings in Italian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-07 tags: [cc, embeddings, fastText, word2vec, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_it_3.4.1_3.0_1646660816126.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_it_3.4.1_3.0_1646660816126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L4_768", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L4_768", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L4_768').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L4_768_embeddings sentence [0.4034431576728821, 0.08385054022073746, 0.08... I hate cancer [-0.2262536883354187, 0.02650507353246212, -0.... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L4_768| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1 --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from dkasti) author: John Snow Labs name: xlmroberta_ner_dkasti_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `dkasti`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_dkasti_base_finetuned_panx_de_4.1.0_3.0_1660432395543.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_dkasti_base_finetuned_panx_de_4.1.0_3.0_1660432395543.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_dkasti_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_dkasti_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_dkasti_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/dkasti/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_de_4.2.0_3.0_1664118970719.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_de_4.2.0_3.0_1664118970719.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate English to Philippine languages Pipeline author: John Snow Labs name: translate_en_phi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, phi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `phi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_phi_xx_2.7.0_2.4_1609691790773.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_phi_xx_2.7.0_2.4_1609691790773.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_phi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_phi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.phi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_phi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_puppies_classify ViTForImageClassification from cherrypaca author: John Snow Labs name: image_classifier_vit_puppies_classify date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_puppies_classify` is a English model originally trained by cherrypaca. ## Predicted Entities `corgi`, `husky`, `pomeranian` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_puppies_classify_en_4.1.0_3.0_1660170671838.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_puppies_classify_en_4.1.0_3.0_1660170671838.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_puppies_classify", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_puppies_classify", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_puppies_classify| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Smaller BERT Embeddings (L-2_H-256_A-4) author: John Snow Labs name: small_bert_L2_256 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L2_256_en_2.6.0_2.4_1598344391697.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L2_256_en_2.6.0_2.4_1598344391697.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L2_256", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L2_256", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L2_256').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L2_256_embeddings I [-1.712011456489563, -1.076645851135254, 0.697... love [-1.1276499032974243, -0.9930340647697449, 1.5... NLP [-0.3206934928894043, 0.03202249854803085, 1.4... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L2_256| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|256| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1 --- layout: model title: English image_classifier_vit_food ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_food date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_food` is a English model originally trained by nateraw. ## Predicted Entities `grilled_cheese_sandwich`, `edamame`, `onion_rings`, `french_onion_soup`, `french_fries`, `creme_brulee`, `lobster_roll_sandwich`, `bruschetta`, `breakfast_burrito`, `caprese_salad`, `churros`, `omelette`, `club_sandwich`, `chocolate_mousse`, `nachos`, `bread_pudding`, `steak`, `hummus`, `panna_cotta`, `filet_mignon`, `sashimi`, `hot_and_sour_soup`, `cannoli`, `ravioli`, `samosa`, `grilled_salmon`, `lobster_bisque`, `seaweed_salad`, `macaroni_and_cheese`, `fish_and_chips`, `caesar_salad`, `dumplings`, `baby_back_ribs`, `fried_rice`, `oysters`, `peking_duck`, `guacamole`, `greek_salad`, `donuts`, `risotto`, `escargots`, `crab_cakes`, `waffles`, `carrot_cake`, `prime_rib`, `tuna_tartare`, `pho`, `chocolate_cake`, `bibimbap`, `fried_calamari`, `spaghetti_bolognese`, `gnocchi`, `chicken_quesadilla`, `frozen_yogurt`, `apple_pie`, `baklava`, `pulled_pork_sandwich`, `clam_chowder`, `eggs_benedict`, `lasagna`, `ceviche`, `paella`, `foie_gras`, `spring_rolls`, `falafel`, `miso_soup`, `pork_chop`, `ramen`, `pad_thai`, `garlic_bread`, `macarons`, `ice_cream`, `mussels`, `chicken_wings`, `pancakes`, `gyoza`, `poutine`, `croque_madame`, `pizza`, `cheese_plate`, `beignets`, `huevos_rancheros`, `french_toast`, `sushi`, `takoyaki`, `spaghetti_carbonara`, `beef_tartare`, `scallops`, `cup_cakes`, `tacos`, `deviled_eggs`, `beet_salad`, `tiramisu`, `cheesecake`, `strawberry_shortcake`, `beef_carpaccio`, `hamburger`, `red_velvet_cake`, `hot_dog`, `shrimp_and_grits`, `chicken_curry` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_food_en_4.1.0_3.0_1660167590552.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_food_en_4.1.0_3.0_1660167590552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_food", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_food", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_food| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.2 MB| --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_10_h_256 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-10_H-256` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_256_zh_4.2.4_3.0_1670021467871.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_10_h_256_zh_4.2.4_3.0_1670021467871.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_256","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_10_h_256","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_10_h_256| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|51.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-10_H-256 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Extraction of Clinical Abbreviations and Acronyms author: John Snow Labs name: ner_abbreviation_emb_clinical_medium date: 2023-05-12 tags: [ner, abbreviation, acronym, en, clinical, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to extract clinical abbreviations and acronyms in text. ## Predicted Entities `ABBR` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ABBREVIATION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_abbreviation_emb_clinical_medium_en_4.4.1_3.0_1683884147067.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_abbreviation_emb_clinical_medium_en_4.4.1_3.0_1683884147067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_abbreviation_emb_clinical_medium", "en", "clinical/models")\ .setInputCols(["sentence", "token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["Gravid with estimated fetal weight of 6-6/12 pounds. LOWER EXTREMITIES: No edema. LABORATORY DATA: Laboratory tests include a CBC which is normal. Blood Type: AB positive. Rubella: Immune. VDRL: Nonreactive. Hepatitis C surface antigen: Negative. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_abbreviation_emb_clinical_medium", "en", "clinical/models") .setInputCols(Array("sentence", "token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter)) val data = Seq("Gravid with estimated fetal weight of 6-6/12 pounds. LOWER EXTREMITIES: No edema. LABORATORY DATA: Laboratory tests include a CBC which is normal. Blood Type: AB positive. Rubella: Immune. VDRL: Nonreactive. Hepatitis C surface antigen: Negative. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-----+-----+---+---------+ |chunk|begin|end|ner_label| +-----+-----+---+---------+ |CBC |126 |128|ABBR | |AB |159 |160|ABBR | |VDRL |189 |192|ABBR | |HIV |247 |249|ABBR | +-----+-----+---+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_abbreviation_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.8 MB| ## References Trained on the in-house dataset. ## Benchmarking ```bash label precision recall f1-score support ABBR 0.94 0.95 0.95 620 micro-avg 0.94 0.95 0.95 620 macro-avg 0.94 0.95 0.95 620 weighted-avg 0.94 0.95 0.95 620 ``` --- layout: model title: Portuguese BertForTokenClassification Cased model (from pucpr) author: John Snow Labs name: bert_token_classifier_clinicalnerpt_disorder date: 2022-11-30 tags: [pt, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: pt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `clinicalnerpt-disorder` is a Portuguese model originally trained by `pucpr`. ## Predicted Entities `Disorder` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_disorder_pt_4.2.4_3.0_1669822474674.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_clinicalnerpt_disorder_pt_4.2.4_3.0_1669822474674.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_disorder","pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_clinicalnerpt_disorder","pt") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_clinicalnerpt_disorder| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|pt| |Size:|665.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/pucpr/clinicalnerpt-disorder - https://www.aclweb.org/anthology/2020.clinicalnlp-1.7/ - https://github.com/HAILab-PUCPR/SemClinBr - https://github.com/HAILab-PUCPR/BioBERTpt --- layout: model title: Part of Speech for Polish author: John Snow Labs name: pos_lfg date: 2021-03-23 tags: [pl, open_source] supported: true task: Part of Speech Tagging language: pl edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_lfg_pl_2.7.5_2.4_1616510144592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_lfg_pl_2.7.5_2.4_1616510144592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_lfg", "pl")\ .setInputCols(["document", "token"])\ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['Zarobki wszystkich nauczycieli będą rosły co rok .']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_lfg", "pl") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("Zarobki wszystkich nauczycieli będą rosły co rok .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Zarobki wszystkich nauczycieli będą rosły co rok .""] token_df = nlu.load('pl.pos.lfg').predict(text) token_df ```

## Results ```bash +--------------------------------------------------+----------------------------------------------+ |text |result | +--------------------------------------------------+----------------------------------------------+ |Zarobki wszystkich nauczycieli będą rosły co rok .|[NOUN, DET, NOUN, AUX, VERB, ADP, NOUN, PUNCT]| +--------------------------------------------------+----------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_lfg| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|pl| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.93 | 0.90 | 0.92 | 830 | | ADP | 0.98 | 0.99 | 0.99 | 1097 | | ADV | 0.91 | 0.94 | 0.93 | 589 | | AUX | 0.93 | 0.95 | 0.94 | 429 | | CCONJ | 0.98 | 0.99 | 0.98 | 354 | | DET | 0.94 | 0.91 | 0.93 | 324 | | INTJ | 0.67 | 0.33 | 0.44 | 6 | | NOUN | 0.93 | 0.95 | 0.94 | 2457 | | NUM | 0.92 | 0.94 | 0.93 | 90 | | PART | 0.99 | 0.95 | 0.97 | 597 | | PRON | 0.98 | 0.97 | 0.97 | 986 | | PROPN | 0.92 | 0.87 | 0.89 | 470 | | PUNCT | 1.00 | 1.00 | 1.00 | 2555 | | SCONJ | 0.97 | 0.99 | 0.98 | 141 | | VERB | 0.96 | 0.96 | 0.96 | 2187 | | accuracy | | | 0.96 | 13112 | | macro avg | 0.93 | 0.91 | 0.92 | 13112 | | weighted avg | 0.96 | 0.96 | 0.96 | 13112 | | weighted avg | 0.92 | 0.92 | 0.92 | 139697 | ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from victorlee071200) author: John Snow Labs name: bert_qa_victorlee071200_base_cased_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-finetuned-squad` is a English model originally trained by `victorlee071200`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_victorlee071200_base_cased_finetuned_squad_en_4.0.0_3.0_1657182882040.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_victorlee071200_base_cased_finetuned_squad_en_4.0.0_3.0_1657182882040.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_victorlee071200_base_cased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_victorlee071200_base_cased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_victorlee071200_base_cased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/victorlee071200/bert-base-cased-finetuned-squad --- layout: model title: Financial ORG, PRODUCT and ALIAS NER (Small) author: John Snow Labs name: finner_orgs_prods_alias date: 2022-08-17 tags: [en, finance, ner, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a large Named Entity Recognition model, trained with a subset of generic conLL, financial and legal conll, ontonotes and several in-house corpora, to detect Organizations, Products and Aliases of Companies. ## Predicted Entities `ORG`, `PROD`, `ALIAS` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINNER_ORGPROD){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_orgs_prods_alias_en_1.0.0_3.2_1660733832114.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_orgs_prods_alias_en_1.0.0_3.2_1660733832114.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------+---------+----------+ |chunk |ner_label|confidence| +------------------------------+---------+----------+ |Spell Security Private Limited|ORG |0.8475 | |Spell Security |ALIAS |0.8871 | |Policy Compliance |PRODUCT |0.7991 | +------------------------------+---------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_orgs_prods_alias| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.7 MB| ## References ConLL-2003, FinSec ConLL, a subset of Ontonotes, In-house corpora ## Benchmarking ```bash label tp fp fn prec rec f1 I-ORG 12853 2621 2685 0.8306191 0.82719785 0.828905 B-PRODUCT 2306 697 932 0.76789874 0.712168 0.7389841 I-ALIAS 14 6 13 0.7 0.5185185 0.59574467 B-ORG 8967 2078 2311 0.81186056 0.79508775 0.80338657 I-PRODUCT 2336 803 1091 0.74418604 0.68164575 0.7115443 B-ALIAS 76 14 22 0.84444445 0.7755102 0.80851066 Macro-average 26552 6219 7054 0.78316814 0.7183547 0.7493626 Micro-average 26552 6219 7054 0.8102285 0.790097 0.80003613 ``` --- layout: model title: Cyberbullying Classifier Pipeline in Turkish texts author: John Snow Labs name: classifierdl_berturk_cyberbullying_pipeline date: 2021-08-13 tags: [tr, cyberbullying, pipeline, open_source] task: Pipeline Public language: tr edition: Spark NLP 3.1.3 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pre-trained pipeline Identifies whether a Turkish text contains cyberbullying or not. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_berturk_cyberbullying_pipeline_tr_3.1.3_2.4_1628848526053.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_berturk_cyberbullying_pipeline_tr_3.1.3_2.4_1628848526053.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_berturk_cyberbullying_pipeline", "tr") result = pipeline.fullAnnotate("""Gidişin olsun, dönüşün olmasın inşallah senin..""") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_berturk_cyberbulling_pipeline", "tr") val result = pipeline.fullAnnotate("Gidişin olsun, dönüşün olmasın inşallah senin..")(0) ```

## Results ```bash ["Negative"] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_berturk_cyberbullying_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.1.3+| |License:|Open Source| |Edition:|Official| |Language:|tr| ## Included Models - DocumentAssembler - TokenizerModel - NormalizerModel - StopWordsCleaner - LemmatizerModel - BertEmbeddings - SentenceEmbeddings - ClassifierDLModel --- layout: model title: Legal Intellectual Property Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_intellectual_property_bert date: 2023-03-05 tags: [en, legal, classification, clauses, intellectual_property, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Intellectual_Property` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Intellectual_Property`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_intellectual_property_bert_en_1.0.0_3.0_1678050714983.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_intellectual_property_bert_en_1.0.0_3.0_1678050714983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_intellectual_property_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Intellectual_Property]| |[Other]| |[Other]| |[Intellectual_Property]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_intellectual_property_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Intellectual_Property 0.98 0.96 0.97 52 Other 0.97 0.99 0.98 73 accuracy - - 0.98 125 macro-avg 0.98 0.97 0.98 125 weighted-avg 0.98 0.98 0.98 125 ``` --- layout: model title: Legal Witnesseth Clause Binary Classifier author: John Snow Labs name: legclf_witnesseth_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `witnesseth` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `witnesseth` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_witnesseth_clause_en_1.0.0_3.2_1660124134557.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_witnesseth_clause_en_1.0.0_3.2_1660124134557.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_witnesseth_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[witnesseth]| |[other]| |[other]| |[witnesseth]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_witnesseth_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.94 0.95 125 witnesseth 0.88 0.89 0.88 55 accuracy - - 0.93 180 macro-avg 0.91 0.92 0.92 180 weighted-avg 0.93 0.93 0.93 180 ``` --- layout: model title: Fast Neural Machine Translation Model from Lozi to English author: John Snow Labs name: opus_mt_loz_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, loz, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `loz` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_loz_en_xx_2.7.0_2.4_1609170483069.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_loz_en_xx_2.7.0_2.4_1609170483069.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_loz_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Solvency Clause Binary Classifier author: John Snow Labs name: legclf_solvency_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `solvency` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `solvency` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_solvency_clause_en_1.0.0_3.2_1660123006192.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_solvency_clause_en_1.0.0_3.2_1660123006192.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_solvency_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[solvency]| |[other]| |[other]| |[solvency]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_solvency_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support insolvency 0.98 0.98 0.98 43 other 0.99 0.99 0.99 101 accuracy - - 0.99 144 macro-avg 0.98 0.98 0.98 144 weighted-avg 0.99 0.99 0.99 144 ``` --- layout: model title: Explain Document Pipeline for Norwegian (Bokmal) author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, norwegian_bokmal, explain_document_md, pipeline, "no"] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: "no" edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_no_3.0.0_3.0_1616435687010.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_no_3.0.0_3.0_1616435687010.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'no') annotations = pipeline.fullAnnotate(""Hei fra John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "no") val result = pipeline.fullAnnotate("Hei fra John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hei fra John Snow Labs! ""] result_df = nlu.load('no.explain.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:--------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Hei fra John Snow Labs! '] | ['Hei fra John Snow Labs!'] | ['Hei', 'fra', 'John', 'Snow', 'Labs!'] | ['Hei', 'fra', 'John', 'Snow', 'Labs!'] | ['PROPN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.1868100017309188,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|no| --- layout: model title: Clinical Findings to UMLS Code Pipeline author: John Snow Labs name: umls_clinical_findings_resolver_pipeline date: 2022-07-26 tags: [en, licensed, umls, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps entities (Clinical Findings) with their corresponding UMLS CUI codes. You’ll just feed your text and it will return the corresponding UMLS codes. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/umls_clinical_findings_resolver_pipeline_en_4.0.0_3.0_1658822255140.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/umls_clinical_findings_resolver_pipeline_en_4.0.0_3.0_1658822255140.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline= PretrainedPipeline("umls_clinical_findings_resolver_pipeline", "en", "clinical/models") pipeline.annotate("HTG-induced pancreatitis associated with an acute hepatitis, and obesity") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline= PretrainedPipeline("umls_clinical_findings_resolver_pipeline", "en", "clinical/models") val pipeline.annotate("HTG-induced pancreatitis associated with an acute hepatitis, and obesity") ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.umls_clinical_findings_resolver").predict("""HTG-induced pancreatitis associated with an acute hepatitis, and obesity""") ```

## Results ```bash +------------------------+---------+---------+ |chunk |ner_label|umls_code| +------------------------+---------+---------+ |HTG-induced pancreatitis|PROBLEM |C1963198 | |an acute hepatitis |PROBLEM |C4750596 | |obesity |PROBLEM |C1963185 | +------------------------+---------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|umls_clinical_findings_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|4.3 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger --- layout: model title: English asr_english_filipino_wav2vec2_l_xls_r_test_03 TFWav2Vec2ForCTC from Khalsuu author: John Snow Labs name: asr_english_filipino_wav2vec2_l_xls_r_test_03 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_english_filipino_wav2vec2_l_xls_r_test_03` is a English model originally trained by Khalsuu. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_english_filipino_wav2vec2_l_xls_r_test_03_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_english_filipino_wav2vec2_l_xls_r_test_03_en_4.2.0_3.0_1664108041739.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_english_filipino_wav2vec2_l_xls_r_test_03_en_4.2.0_3.0_1664108041739.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_english_filipino_wav2vec2_l_xls_r_test_03", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_english_filipino_wav2vec2_l_xls_r_test_03", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_english_filipino_wav2vec2_l_xls_r_test_03| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from skr3178) author: John Snow Labs name: xlmroberta_ner_skr3178_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `skr3178`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_skr3178_base_finetuned_panx_all_xx_4.1.0_3.0_1660428924987.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_skr3178_base_finetuned_panx_all_xx_4.1.0_3.0_1660428924987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_skr3178_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_skr3178_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_skr3178_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/skr3178/xlm-roberta-base-finetuned-panx-all --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_medium_pretrained_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-medium-pretrained-finetuned-squad` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_pretrained_finetuned_squad_en_4.0.0_3.0_1654183730265.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_pretrained_finetuned_squad_en_4.0.0_3.0_1654183730265.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_medium_pretrained_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_medium_pretrained_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.medium_finetuned.by_anas-awadalla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_medium_pretrained_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|154.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-medium-pretrained-finetuned-squad --- layout: model title: Legal Notices Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_notices_bert date: 2023-03-05 tags: [en, legal, classification, clauses, notices, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Notices` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Notices`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_notices_bert_en_1.0.0_3.0_1678049992578.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_notices_bert_en_1.0.0_3.0_1678049992578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_notices_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Notices]| |[Other]| |[Other]| |[Notices]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_notices_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Notices 0.98 0.97 0.97 301 Other 0.97 0.98 0.97 337 accuracy - - 0.97 638 macro-avg 0.97 0.97 0.97 638 weighted-avg 0.97 0.97 0.97 638 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from dbernsohn) author: John Snow Labs name: t5_measurement_time date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5_measurement_time` is a English model originally trained by `dbernsohn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_measurement_time_en_4.3.0_3.0_1675156797621.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_measurement_time_en_4.3.0_3.0_1675156797621.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_measurement_time","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_measurement_time","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_measurement_time| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|278.9 MB| ## References - https://huggingface.co/dbernsohn/t5_measurement_time - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://www.tensorflow.org/datasets/catalog/math_dataset#mathdatasetmeasurement_time - https://github.com/DorBernsohn/CodeLM/tree/main/MathLM - https://www.linkedin.com/in/dor-bernsohn-70b2b1146/ --- layout: model title: Legal Adoption Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_adoption_agreement_bert date: 2023-01-29 tags: [en, legal, classification, adoption, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_adoption_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `adoption-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `adoption-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_adoption_agreement_bert_en_1.0.0_3.0_1674990271078.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_adoption_agreement_bert_en_1.0.0_3.0_1674990271078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_adoption_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[adoption-agreement]| |[other]| |[other]| |[adoption-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_adoption_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.91 0.97 0.94 98 standstill-agreement 0.93 0.82 0.87 51 accuracy - - 0.92 149 macro-avg 0.92 0.90 0.91 149 weighted-avg 0.92 0.92 0.92 149 ``` --- layout: model title: Translate English to Urdu Pipeline author: John Snow Labs name: translate_en_ur date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ur, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ur` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ur_xx_2.7.0_2.4_1609690466742.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ur_xx_2.7.0_2.4_1609690466742.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ur", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ur", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ur').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ur| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from ktrapeznikov) author: John Snow Labs name: bert_qa_scibert_scivocab_uncased_squad_v2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `scibert_scivocab_uncased_squad_v2` is a English model orginally trained by `ktrapeznikov`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_scivocab_uncased_squad_v2_en_4.0.0_3.0_1654189468977.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_scibert_scivocab_uncased_squad_v2_en_4.0.0_3.0_1654189468977.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_scibert_scivocab_uncased_squad_v2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_scibert_scivocab_uncased_squad_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.scibert.uncased_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_scibert_scivocab_uncased_squad_v2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ktrapeznikov/scibert_scivocab_uncased_squad_v2 - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Word2Vec Embeddings in Volapük (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, vo, open_source] task: Embeddings language: vo edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_vo_3.4.1_3.0_1647466940019.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_vo_3.4.1_3.0_1647466940019.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_kr_electra_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ko| |Size:|124.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/snunlp/KR-ELECTRA-generator - https://github.com/google-research/electra - https://github.com/google-research/electra - https://bitbucket.org/eunjeon/mecab-ko-dic/src/master/ - https://drive.google.com/file/d/1L_yKEDaXM_yDLwHm5QrXAncQZiMN3BBU/view?usp=sharing - https://github.com/monologg/KoELECTRA - https://github.com/snunlp/KR-ELECTRA - https://github.com/monologg/KoELECTRA --- layout: model title: Translate Kabyle to English Pipeline author: John Snow Labs name: translate_kab_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, kab, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `kab` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_kab_en_xx_2.7.0_2.4_1609690292388.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_kab_en_xx_2.7.0_2.4_1609690292388.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_kab_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_kab_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.kab.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_kab_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Sangita) author: John Snow Labs name: distilbert_qa_sangita_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Sangita`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sangita_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769187333.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sangita_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769187333.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sangita_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sangita_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sangita_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Sangita/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Base Salary Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_base_salary_bert date: 2023-03-05 tags: [en, legal, classification, clauses, base_salary, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Base_Salary` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Base_Salary`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_base_salary_bert_en_1.0.0_3.0_1678050573214.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_base_salary_bert_en_1.0.0_3.0_1678050573214.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_base_salary_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Base_Salary]| |[Other]| |[Other]| |[Base_Salary]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_base_salary_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Base_Salary 1.00 1.00 1.0 77 Other 1.00 1.00 1.0 108 accuracy - - 1.0 185 macro-avg 1.00 1.00 1.0 185 weighted-avg 1.00 1.00 1.0 185 ``` --- layout: model title: English image_classifier_vit_base_patch16_224_finetuned_eurosat ViTForImageClassification from rwang5688 author: John Snow Labs name: image_classifier_vit_base_patch16_224_finetuned_eurosat date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_finetuned_eurosat` is a English model originally trained by rwang5688. ## Predicted Entities `Residential`, `AnnualCrop`, `Highway`, `Pasture`, `SeaLake`, `Industrial`, `HerbaceousVegetation`, `River`, `PermanentCrop`, `Forest` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_finetuned_eurosat_en_4.1.0_3.0_1660171928095.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_finetuned_eurosat_en_4.1.0_3.0_1660171928095.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_finetuned_eurosat", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_finetuned_eurosat", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_finetuned_eurosat| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: NER Pipeline for German author: John Snow Labs name: xlm_roberta_large_token_classifier_conll03_pipeline date: 2022-06-27 tags: [german, roberta, xlm, ner, conll03, de, open_source] task: Named Entity Recognition language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_large_token_classifier_conll03_de](https://nlp.johnsnowlabs.com/2021/12/25/xlm_roberta_large_token_classifier_conll03_de.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_conll03_pipeline_de_4.0.0_3.0_1656373648807.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_conll03_pipeline_de_4.0.0_3.0_1656373648807.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("xlm_roberta_large_token_classifier_conll03_pipeline", lang = "de") pipeline.annotate("Ibser begann seine Karriere beim ASK Ebreichsdorf. 2004 wechselte er zu Admira Wacker Mödling, wo er auch in der Akademie spielte.") ``` ```scala val pipeline = new PretrainedPipeline("xlm_roberta_large_token_classifier_conll03_pipeline", lang = "de") pipeline.annotate("Ibser begann seine Karriere beim ASK Ebreichsdorf. 2004 wechselte er zu Admira Wacker Mödling, wo er auch in der Akademie spielte.") ```

## Results ```bash +----------------------+---------+ |chunk |ner_label| +----------------------+---------+ |Ibser |PER | |ASK Ebreichsdorf |ORG | |Admira Wacker Mödling |ORG | +----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_large_token_classifier_conll03_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.8 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - XlmRoBertaForTokenClassification - NerConverter - Finisher --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_nh32 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-nh32` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nh32_en_4.3.0_3.0_1675113289211.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nh32_en_4.3.0_3.0_1675113289211.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_nh32","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_nh32","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_nh32| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|745.5 MB| ## References - https://huggingface.co/google/t5-efficient-base-nh32 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-512-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2_en_4.0.0_3.0_1655732960556.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2_en_4.0.0_3.0_1655732960556.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_512d_seed_2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_512_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|432.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-512-finetuned-squad-seed-2 --- layout: model title: Fast Neural Machine Translation Model from Turkish to English author: John Snow Labs name: opus_mt_tr_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tr, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tr` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tr_en_xx_2.7.0_2.4_1609169109312.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tr_en_xx_2.7.0_2.4_1609169109312.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tr_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from English to Gun author: John Snow Labs name: opus_mt_en_guw date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, guw, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `guw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_guw_xx_2.7.0_2.4_1609164685066.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_guw_xx_2.7.0_2.4_1609164685066.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_guw| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke TFWav2Vec2ForCTC from logicbloke author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke` is a English model originally trained by logicbloke. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke_en_4.2.0_3.0_1664095691550.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke_en_4.2.0_3.0_1664095691550.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_arabic_by_logicbloke| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline to Detect PHI for Generic Deidentification in Romanian author: John Snow Labs name: ner_deid_generic_pipeline date: 2023-03-09 tags: [deidentification, word2vec, phi, generic, ner, ro, licensed] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_deid_generic](https://nlp.johnsnowlabs.com/2022/07/08/ner_deid_generic_ro_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_pipeline_ro_4.3.0_3.2_1678382243449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_generic_pipeline_ro_4.3.0_3.2_1678382243449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_deid_generic_pipeline", "ro", "clinical/models") text = '''Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui,737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_deid_generic_pipeline", "ro", "clinical/models") val text = "Spitalul Pentru Ochi de Deal, Drumul Oprea Nr. 972 Vaslui,737405 România Tel: +40(235)413773 Data setului de analize: 25 May 2022 15:36:00 Nume si Prenume : BUREAN MARIA, Varsta: 77 Medic : Agota Evelyn Tımar C.N.P : 2450502264401" val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------------------|--------:|------:|:------------|-------------:| | 0 | Spitalul Pentru Ochi de Deal | 0 | 27 | LOCATION | 0.88326 | | 1 | Drumul Oprea Nr. 972 | 30 | 49 | LOCATION | 0.98642 | | 2 | Vaslui,737405 România | 51 | 71 | LOCATION | 0.8018 | | 3 | +40(235)413773 | 78 | 91 | CONTACT | 1 | | 4 | 25 May 2022 | 118 | 128 | DATE | 1 | | 5 | BUREAN MARIA | 157 | 168 | NAME | 0.99965 | | 6 | 77 | 179 | 180 | AGE | 1 | | 7 | Agota Evelyn Tımar | 190 | 207 | NAME | 0.832933 | | 8 | 2450502264401 | 217 | 229 | ID | 1 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_generic_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|ro| |Size:|1.2 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Detect Entities (66-labeled) in General Scope (Few-NERD dataset) author: John Snow Labs name: nerdl_fewnerd_subentity_100d date: 2021-07-22 tags: [ner, en, fewnerd, public, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.1.1 spark_version: 2.4 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained on Few-NERD/inter public dataset and it extracts 66 entities that are in general scope. ## Predicted Entities `building-theater`, `art-other`, `location-bodiesofwater`, `other-god`, `organization-politicalparty`, `product-other`, `building-sportsfacility`, `building-restaurant`, `organization-sportsleague`, `event-election`, `organization-media/newspaper`, `product-software`, `other-educationaldegree`, `person-politician`, `person-soldier`, `other-disease`, `product-airplane`, `person-athlete`, `location-mountain`, `organization-company`, `other-biologything`, `location-other`, `other-livingthing`, `person-actor`, `organization-other`, `event-protest`, `art-film`, `other-award`, `other-astronomything`, `building-airport`, `product-food`, `person-other`, `event-disaster`, `product-weapon`, `event-sportsevent`, `location-park`, `product-ship`, `building-library`, `art-painting`, `building-other`, `other-currency`, `organization-education`, `person-scholar`, `organization-showorganization`, `person-artist/author`, `product-train`, `location-GPE`, `product-car`, `art-writtenart`, `event-attack/battle/war/militaryconflict`, `other-law`, `other-medical`, `organization-sportsteam`, `art-broadcastprogram`, `art-music`, `organization-government/governmentagency`, `other-language`, `event-other`, `person-director`, `other-chemicalthing`, `product-game`, `organization-religion`, `location-road/railway/highway/transit`, `location-island`, `building-hotel`, `building-hospital` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_FEW_NERD/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_FewNERD.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_en_3.1.1_2.4_1626970707030.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/nerdl_fewnerd_subentity_100d_en_3.1.1_2.4_1626970707030.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("glove_100d", "en")\ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("nerdl_fewnerd_subentity_100d") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(['document', 'token', 'ner']) \ .setOutputCol('ner_chunk') nlp_pipeline = Pipeline(stages=[document_assembler, sentencer, tokenizer, embeddings, ner, ner_converter]) l_model = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = l_model.fullAnnotate("""12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.""") ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("glove_100d", "en") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("nerdl_fewnerd_subentity_100d") .setInputCols(Array("sentence", "token", "embeddings")).setOutputCol("ner") val ner_converter = NerConverter.setInputCols(Array("document", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, ner, ner_converter)) val data = Seq("12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.fewnerd_subentity").predict("""12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.""") ```

## Results ```bash +-----------------------+----------------------------+ |chunk |ner_label | +-----------------------+----------------------------+ |Corazones ('12 Hearts')|art-broadcastprogram | |Spanish-language |other-language | |United States |location-GPE | |Telemundo |organization-media/newspaper| |Argentine TV |organization-media/newspaper| |Los Angeles |location-GPE | |Steven Spielberg |person-director | |Cloverfield Paradox |art-film | +-----------------------+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|nerdl_fewnerd_subentity_100d| |Type:|ner| |Compatibility:|Spark NLP 3.1.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Few-NERD:A Few-shot Named Entity Recognition Dataset, author: Ding, Ning and Xu, Guangwei and Chen, Yulin, and Wang, Xiaobin and Han, Xu and Xie, Pengjun and Zheng, Hai-Tao and Liu, Zhiyuan, book title: ACL-IJCNL, 2021. ## Benchmarking ```bash +--------------------+-------+------+-------+-------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +--------------------+-------+------+-------+-------+---------+------+------+ | disaster| 309.0| 114.0| 287.0| 596.0| 0.7305|0.5185|0.6065| | film| 1589.0| 725.0| 810.0| 2399.0| 0.6867|0.6624|0.6743| | mountain| 851.0| 175.0| 431.0| 1282.0| 0.8294|0.6638|0.7374| | currency| 280.0| 66.0| 189.0| 469.0| 0.8092| 0.597|0.6871| | scholar| 31.0| 12.0| 413.0| 444.0| 0.7209|0.0698|0.1273| | island| 829.0| 165.0| 372.0| 1201.0| 0.834|0.6903|0.7554| | politicalparty| 242.0| 86.0| 283.0| 525.0| 0.7378| 0.461|0.5674| | ship| 461.0| 207.0| 311.0| 772.0| 0.6901|0.5972|0.6403| | award| 2234.0| 279.0| 1245.0| 3479.0| 0.889|0.6421|0.7457| | showorganization| 120.0| 201.0| 273.0| 393.0| 0.3738|0.3053|0.3361| | religion| 218.0| 117.0| 415.0| 633.0| 0.6507|0.3444|0.4504| | education| 5788.0| 852.0| 1001.0| 6789.0| 0.8717|0.8526| 0.862| | park| 259.0| 295.0| 176.0| 435.0| 0.4675|0.5954|0.5238| | painting| 0.0| 0.0| 14.0| 14.0| 0.0| 0.0| 0.0| | hotel| 570.0| 150.0| 254.0| 824.0| 0.7917|0.6917|0.7383| | library| 218.0| 92.0| 134.0| 352.0| 0.7032|0.6193|0.6586| | livingthing| 576.0| 280.0| 312.0| 888.0| 0.6729|0.6486|0.6606| | educationaldegree| 189.0| 31.0| 47.0| 236.0| 0.8591|0.8008|0.8289| | director| 673.0| 227.0| 507.0| 1180.0| 0.7478|0.5703|0.6471| | food| 474.0| 375.0| 341.0| 815.0| 0.5583|0.5816|0.5697| | athlete| 1181.0| 529.0| 540.0| 1721.0| 0.6906|0.6862|0.6884| | software| 922.0| 460.0| 493.0| 1415.0| 0.6671|0.6516|0.6593| | protest| 162.0| 212.0| 275.0| 437.0| 0.4332|0.3707|0.3995| | other|12555.0|7510.0|14369.0|26924.0| 0.6257|0.4663|0.5344| | sportsleague| 1439.0| 654.0| 842.0| 2281.0| 0.6875|0.6309| 0.658| | airplane| 1295.0| 442.0| 463.0| 1758.0| 0.7455|0.7366|0.7411| | train| 135.0| 111.0| 198.0| 333.0| 0.5488|0.4054|0.4663| | biologything| 1574.0| 625.0| 924.0| 2498.0| 0.7158|0.6301|0.6702| | politician| 3107.0|1545.0| 1688.0| 4795.0| 0.6679| 0.648|0.6578| | music| 419.0| 211.0| 182.0| 601.0| 0.6651|0.6972|0.6807| |government/govern...| 564.0| 656.0| 511.0| 1075.0| 0.4623|0.5247|0.4915| | media/newspaper| 1600.0|1072.0| 893.0| 2493.0| 0.5988|0.6418|0.6196| | actor| 674.0| 161.0| 274.0| 948.0| 0.8072| 0.711| 0.756| | language| 698.0| 226.0| 335.0| 1033.0| 0.7554|0.6757|0.7133| | chemicalthing| 592.0| 231.0| 687.0| 1279.0| 0.7193|0.4629|0.5633| | sportsfacility| 870.0| 334.0| 291.0| 1161.0| 0.7226|0.7494|0.7357| | hospital| 226.0| 472.0| 49.0| 275.0| 0.3238|0.8218|0.4645| | writtenart| 297.0| 203.0| 450.0| 747.0| 0.594|0.3976|0.4763| |road/railway/high...| 3238.0| 926.0| 1063.0| 4301.0| 0.7776|0.7528| 0.765| | election| 13.0| 13.0| 127.0| 140.0| 0.5|0.0929|0.1566| | soldier| 623.0| 537.0| 559.0| 1182.0| 0.5371|0.5271| 0.532| | god| 332.0| 157.0| 414.0| 746.0| 0.6789| 0.445|0.5377| | astronomything| 1120.0| 353.0| 232.0| 1352.0| 0.7604|0.8284|0.7929| |attack/battle/war...| 2516.0| 444.0| 590.0| 3106.0| 0.85| 0.81|0.8295| | broadcastprogram| 1056.0| 762.0| 811.0| 1867.0| 0.5809|0.5656|0.5731| | airport| 857.0| 96.0| 112.0| 969.0| 0.8993|0.8844|0.8918| | theater| 72.0| 31.0| 119.0| 191.0| 0.699| 0.377|0.4898| | weapon| 303.0| 190.0| 237.0| 540.0| 0.6146|0.5611|0.5866| | company| 5849.0|2632.0| 2570.0| 8419.0| 0.6897|0.6947|0.6922| | car| 413.0| 293.0| 207.0| 620.0| 0.585|0.6661|0.6229| | artist/author| 4172.0|1953.0| 1777.0| 5949.0| 0.6811|0.7013|0.6911| | medical| 94.0| 112.0| 192.0| 286.0| 0.4563|0.3287|0.3821| | disease| 1009.0| 476.0| 447.0| 1456.0| 0.6795| 0.693|0.6862| | game| 141.0| 120.0| 264.0| 405.0| 0.5402|0.3481|0.4234| | sportsevent| 1042.0| 553.0| 552.0| 1594.0| 0.6533|0.6537|0.6535| | sportsteam| 3657.0|1133.0| 1301.0| 4958.0| 0.7635|0.7376|0.7503| | restaurant| 285.0| 444.0| 201.0| 486.0| 0.3909|0.5864|0.4691| | bodiesofwater| 314.0| 91.0| 343.0| 657.0| 0.7753|0.4779|0.5913| | law| 1626.0| 583.0| 329.0| 1955.0| 0.7361|0.8317| 0.781| | GPE|22173.0|5585.0| 3839.0|26012.0| 0.7988|0.8524|0.8247| +--------------------+-------+------+-------+-------+---------+------+------+ +-----------------+ | macro| +-----------------+ |0.608599546406531| +-----------------+ +-----------------+ | micro| +-----------------+ |0.684720504256685| +-----------------+ ``` --- layout: model title: Bulgarian RobertaForMaskedLM Small Cased model (from iarfmoose) author: John Snow Labs name: roberta_embeddings_small_bulgarian date: 2022-12-12 tags: [bg, open_source, roberta_embeddings, robertaformaskedlm] task: Embeddings language: bg edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-small-bulgarian` is a Bulgarian model originally trained by `iarfmoose`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_small_bulgarian_bg_4.2.4_3.0_1670859644104.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_small_bulgarian_bg_4.2.4_3.0_1670859644104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_small_bulgarian","bg") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_loaded = RoBertaEmbeddings.pretrained("roberta_embeddings_small_bulgarian","bg") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_small_bulgarian| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|bg| |Size:|314.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/iarfmoose/roberta-small-bulgarian - https://arxiv.org/abs/1907.11692 - https://oscar-corpus.com/ - https://wortschatz.uni-leipzig.de/en/download/bulgarian - https://wortschatz.uni-leipzig.de/en/download/bulgarian --- layout: model title: English image_classifier_vit_vision_transformers_spain_or_italy_fan ViTForImageClassification from jeffboudier author: John Snow Labs name: image_classifier_vit_vision_transformers_spain_or_italy_fan date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vision_transformers_spain_or_italy_fan` is a English model originally trained by jeffboudier. ## Predicted Entities `italy soccer fan`, `spain soccer fan` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformers_spain_or_italy_fan_en_4.1.0_3.0_1660166700234.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vision_transformers_spain_or_italy_fan_en_4.1.0_3.0_1660166700234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vision_transformers_spain_or_italy_fan", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vision_transformers_spain_or_italy_fan", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vision_transformers_spain_or_italy_fan| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: ICD10CM Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_clinical_en_3.0.0_3.0_1617355382919.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_clinical_en_3.0.0_3.0_1617355382919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... icd10cm_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_clinical", "en", "clinical/models") \ .setInputCols(["ner_token", "chunk_embeddings"]) \ .setOutputCol("icd10cm_code") \ .setDistanceFunction("COSINE") \ .setNeighbours(5) pipeline_icd10cm = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, chunk_tokenizer, icd10cm_resolution]) pipeline_model = pipeline_icd10cm.fit(spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG."""]]).toDF("text")) result = pipeline_model.transform(data) ``` ```scala ... val icd10cm_resolution = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_clinical", "en", "clinical/models") .setInputCols("ner_token", "chunk_embeddings") .setOutputCol("icd10cm_code") .setDistanceFunction("COSINE") .setNeighbours(5) val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, chunk_tokenizer, icd10cm_resolution)) val data = Seq("A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting. Two weeks prior to presentation, she was treated with a five-day course of amoxicillin for a respiratory tract infection. She was on metformin, glipizide, and dapagliflozin for T2DM and atorvastatin and gemfibrozil for HTG.").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash | | chunk | entity | resolved_text | code | cms | |---|-----------------------------|-----------|----------------------------------------------------|--------|---------------------------------------------------| | 0 | T2DM), | PROBLEM | Type 2 diabetes mellitus with diabetic nephrop... | E1121 | Type 2 diabetes mellitus with diabetic nephrop... | | 1 | T2DM | PROBLEM | Type 2 diabetes mellitus with diabetic nephrop... | E1121 | Type 2 diabetes mellitus with diabetic nephrop... | | 2 | polydipsia | PROBLEM | Polydipsia | R631 | Polydipsia:::Anhedonia:::Galactorrhea | | 3 | interference from turbidity | PROBLEM | Non-working side interference | M2656 | Non-working side interference:::Hemoglobinuria... | | 4 | polyuria | PROBLEM | Other polyuria | R358 | Other polyuria:::Polydipsia:::Generalized edem... | | 5 | lipemia | PROBLEM | Glycosuria | R81 | Glycosuria:::Pure hyperglyceridemia:::Hyperchy... | | 6 | starvation ketosis | PROBLEM | Propionic acidemia | E71121 | Propionic acidemia:::Bartter's syndrome:::Hypo... | | 7 | HTG | PROBLEM | Pure hyperglyceridemia | E781 | Pure hyperglyceridemia:::Familial hypercholest... | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_icd10cm_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[icd10cm]| |Language:|en| --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from jasonyim2) author: John Snow Labs name: xlmroberta_ner_jasonyim2_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `jasonyim2`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jasonyim2_base_finetuned_panx_de_4.1.0_3.0_1660434335874.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jasonyim2_base_finetuned_panx_de_4.1.0_3.0_1660434335874.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jasonyim2_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jasonyim2_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jasonyim2_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jasonyim2/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: French CamemBert Embeddings (from Ebtihal) author: John Snow Labs name: camembert_embeddings_ArBERTMo date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ArBERTMo` is a French model orginally trained by `Ebtihal`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ArBERTMo_fr_3.4.4_3.0_1653985627290.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_ArBERTMo_fr_3.4.4_3.0_1653985627290.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_ArBERTMo| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Ebtihal/ArBERTMo --- layout: model title: Named Entity Recognition - BERT Mini (OntoNotes) author: John Snow Labs name: onto_small_bert_L4_256 date: 2020-12-05 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [ner, open_source, en] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Onto is a Named Entity Recognition (or NER) model trained on OntoNotes 5.0. It can extract up to 18 entities such as people, places, organizations, money, time, date, etc. This model uses the pretrained `small_bert_L4_256` embeddings model from the `BertEmbeddings` annotator as an input. ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_small_bert_L4_256_en_2.7.0_2.4_1607199231735.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_small_bert_L4_256_en_2.7.0_2.4_1607199231735.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... ner_onto = NerDLModel.pretrained("onto_small_bert_L4_256", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val ner_onto = NerDLModel.pretrained("onto_small_bert_L4_256", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_onto, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.onto.bert.small_l4_256').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PERSON | |October 28, 1955 |DATE | |American |NORP | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PERSON | |May 2014 |DATE | |one |CARDINAL | |1970s and 1980s |DATE | |Seattle |GPE | |Washington |GPE | |Gates |PERSON | |Paul Allen |PERSON | |1975 |DATE | |Albuquerque |GPE | |New Mexico |GPE | |Gates |ORG | |January 2000 |DATE | |the late 1990s |DATE | |Gates |PERSON | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_small_bert_L4_256| |Type:|ner| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source The model is trained based on data from [OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) ## Benchmarking ```bash Micro-average: prec: 0.8617996, rec: 0.85458803, f1: 0.8581787 CoNLL Eval: processed 152728 tokens with 11257 phrases; found: 11191 phrases; correct: 9476. accuracy: 97.05%; 9476 11257 11191 precision: 84.68%; recall: 84.18%; FB1: 84.43 CARDINAL: 771 935 934 precision: 82.55%; recall: 82.46%; FB1: 82.50 934 DATE: 1383 1602 1645 precision: 84.07%; recall: 86.33%; FB1: 85.19 1645 EVENT: 29 63 49 precision: 59.18%; recall: 46.03%; FB1: 51.79 49 FAC: 65 135 100 precision: 65.00%; recall: 48.15%; FB1: 55.32 100 GPE: 2054 2240 2211 precision: 92.90%; recall: 91.70%; FB1: 92.29 2211 LANGUAGE: 10 22 13 precision: 76.92%; recall: 45.45%; FB1: 57.14 13 LAW: 11 40 22 precision: 50.00%; recall: 27.50%; FB1: 35.48 22 LOC: 112 179 186 precision: 60.22%; recall: 62.57%; FB1: 61.37 186 MONEY: 272 314 317 precision: 85.80%; recall: 86.62%; FB1: 86.21 317 NORP: 781 841 856 precision: 91.24%; recall: 92.87%; FB1: 92.04 856 ORDINAL: 172 195 228 precision: 75.44%; recall: 88.21%; FB1: 81.32 228 ORG: 1383 1795 1749 precision: 79.07%; recall: 77.05%; FB1: 78.05 1749 PERCENT: 311 349 346 precision: 89.88%; recall: 89.11%; FB1: 89.50 346 PERSON: 1809 1988 2048 precision: 88.33%; recall: 91.00%; FB1: 89.64 2048 PRODUCT: 34 76 50 precision: 68.00%; recall: 44.74%; FB1: 53.97 50 QUANTITY: 83 105 106 precision: 78.30%; recall: 79.05%; FB1: 78.67 106 TIME: 138 212 228 precision: 60.53%; recall: 65.09%; FB1: 62.73 228 WORK_OF_ART: 58 166 103 precision: 56.31%; recall: 34.94%; FB1: 43.12 103 ``` --- layout: model title: Fast Neural Machine Translation Model from Tsonga to English author: John Snow Labs name: opus_mt_ts_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ts, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ts` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ts_en_xx_2.7.0_2.4_1609163263407.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ts_en_xx_2.7.0_2.4_1609163263407.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ts_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Purpose Clause Binary Classifier author: John Snow Labs name: legclf_purpose_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `purpose` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `purpose` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_purpose_clause_en_1.0.0_3.2_1660123878905.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_purpose_clause_en_1.0.0_3.2_1660123878905.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_purpose_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[purpose]| |[other]| |[other]| |[purpose]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_purpose_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.94 0.98 0.96 93 purpose 0.94 0.84 0.89 37 accuracy - - 0.94 130 macro-avg 0.94 0.91 0.92 130 weighted-avg 0.94 0.94 0.94 130 ``` --- layout: model title: French CamemBert Embeddings (from osanseviero) author: John Snow Labs name: camembert_embeddings_generic_model_test date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model-test` is a French model orginally trained by `osanseviero`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_generic_model_test_fr_3.4.4_3.0_1653991144897.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_generic_model_test_fr_3.4.4_3.0_1653991144897.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_generic_model_test| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/osanseviero/dummy-model-test --- layout: model title: Voice of the Patients (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_wip_emb_clinical_medium date: 2023-05-19 tags: [licensed, clinical, en, ner, vop, patient] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Allergen`, `RaceEthnicity`, `SubstanceQuantity`, `InjuryOrPoisoning`, `Treatment`, `Modifier`, `HealthStatus`, `MedicalDevice`, `TestResult`, `RelationshipStatus`, `Route`, `Measurements`, `AdmissionDischarge`, `Frequency`, `Procedure`, `Symptom`, `Substance`, `Duration`, `ClinicalDept`, `Dosage`, `Disease`, `Vaccine`, `Laterality`, `DateTime`, `Drug`, `Test`, `PsychologicalCondition`, `VitalTest`, `Form`, `Age`, `BodyPart`, `Employment`, `Gender` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_wip_emb_clinical_medium_en_4.4.2_3.0_1684509750529.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_wip_emb_clinical_medium_en_4.4.2_3.0_1684509750529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") # Consider using nlp.SentenceDetector with rules/patterns to get smaller chunks from long sentences sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_legal_bert_base_uncased","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") jsl_ner = finance.NerModel.pretrained("finner_sec_conll", "en", "finance/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("jsl_ner") jsl_ner_converter = nlp.NerConverter() \ .setInputCols(["sentence", "token", "jsl_ner"]) \ .setOutputCol("ner_chunk") jsl_ner_pipeline = nlp.Pipeline().setStages([ documentAssembler, sentence_detector, tokenizer, embeddings, jsl_ner, jsl_ner_converter]) text = """December 2007 SUBORDINATED LOAN AGREEMENT. THIS LOAN AGREEMENT is made on 7th December, 2007 BETWEEN: (1) SILICIUM DE PROVENCE S.A.S., a private company with limited liability, incorporated under the laws of France, whose registered office is situated at Usine de Saint Auban, France, represented by Mr.Frank Wouters, hereinafter referred to as the "Borrower", and ( 2 ) EVERGREEN SOLAR INC., a company incorporated in Delaware, U.S.A., with registered number 2426798, whose registered office is situated at Bartlett Street, Marlboro, Massachusetts, U.S.A. represented by Richard Chleboski, hereinafter referred to as "Lender".""" df = spark.createDataFrame([[text]]).toDF("text") model = jsl_ner_pipeline.fit(df) res = model.transform(df) ```

## Results ```bash +------------------------------------------------+-----+ |ner_chunk |label| +------------------------------------------------+-----+ |SILICIUM DE PROVENCE S.A.S |ORG | |France |LOC | |Usine de Saint Auban |LOC | |France |LOC | |Mr.Frank Wouters |PER | |Borrower |PER | |EVERGREEN SOLAR INC |ORG | |Delaware |LOC | |U.S.A |LOC | |Bartlett Street |LOC | |Marlboro |LOC | |Massachusetts |LOC | |U.S.A |LOC | |Richard Chleboski |PER | |Lender |PER | +------------------------------------------------+-----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_sec_conll| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References Manual annotations, coNll2003 and financial documents obtained from U.S. Security and Exchange Commission (SEC) filings. ## Benchmarking ```bash label tp fp fn prec rec f1 B-LOC 14 6 11 0.7 0.56 0.6222222 I-ORG 59 30 3 0.66292137 0.9516129 0.781457 I-LOC 32 2 22 0.9411765 0.5925926 0.7272727 I-PER 18 4 5 0.8181818 0.7826087 0.8 B-ORG 47 17 5 0.734375 0.90384614 0.8103449 B-PER 211 7 2 0.9678899 0.9906103 0.97911835 Macro-average 381 66 48 0.80409074 0.7968784 0.8004684 Micro-average 381 66 48 0.852349 0.8881119 0.869863 ``` --- layout: model title: Legal Note Purchase Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_note_purchase_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, note_purchase, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_note_purchase_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `note-purchase-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `note-purchase-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_note_purchase_agreement_bert_en_1.0.0_3.0_1669368416923.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_note_purchase_agreement_bert_en_1.0.0_3.0_1669368416923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_note_purchase_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[note-purchase-agreement]| |[other]| |[other]| |[note-purchase-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_note_purchase_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support note-purchase-agreement 0.92 0.79 0.85 28 other 0.87 0.95 0.91 41 accuracy - - 0.88 69 macro-avg 0.89 0.87 0.88 69 weighted-avg 0.89 0.88 0.88 69 ``` --- layout: model title: Detect clinical concepts (jsl_ner_wip_modifier_clinical) author: John Snow Labs name: jsl_ner_wip_modifier_clinical date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect modifiers and other clinical entities using pretrained NER model. ## Predicted Entities `Kidney_Disease`, `Height`, `Family_History_Header`, `RelativeTime`, `Hypertension`, `HDL`, `Alcohol`, `Test`, `Substance`, `Fetus_NewBorn`, `Diet`, `Substance_Quantity`, `Allergen`, `Form`, `Birth_Entity`, `Age`, `Race_Ethnicity`, `Modifier`, `Internal_organ_or_component`, `Hyperlipidemia`, `ImagingFindings`, `Psychological_Condition`, `Triglycerides`, `Cerebrovascular_Disease`, `Obesity`, `Duration`, `Weight`, `Date`, `Test_Result`, `Strength`, `VS_Finding`, `Respiration`, `Social_History_Header`, `Employment`, `Injury_or_Poisoning`, `Medical_History_Header`, `Death_Entity`, `Relationship_Status`, `Oxygen_Therapy`, `Blood_Pressure`, `Gender`, `Section_Header`, `Oncological`, `Drug`, `Labour_Delivery`, `Heart_Disease`, `LDL`, `Medical_Device`, `Temperature`, `Treatment`, `Female_Reproductive_Status`, `Total_Cholesterol`, `Time`, `Disease_Syndrome_Disorder`, `Communicable_Disease`, `EKG_Findings`, `Diabetes`, `Route`, `External_body_part_or_region`, `Pulse`, `Vital_Signs_Header`, `Direction`, `Admission_Discharge`, `Overweight`, `RelativeDate`, `O2_Saturation`, `BMI`, `Vaccine`, `Pregnancy`, `Sexually_Active_or_Sexual_Orientation`, `Procedure`, `Frequency`, `Dosage`, `Symptom`, `Clinical_Dept`, `Smoking` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_modifier_clinical_en_3.0.0_3.0_1617260799422.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_modifier_clinical_en_3.0.0_3.0_1617260799422.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------+--------+ | token| ner| +--------------+--------+ | 宮本|B-PERSON| | 茂|I-PERSON| | 氏| O| | は| O| | 、| O| | 日本| B-GPE| | の| O| | 任天| B-FAC| | 堂| I-FAC| | の| O| | ゲーム| O| |プロデューサー| O| | です| O| | 。| O| +--------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ud_gsd_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ja| |Dependencies:|japanese_cc_300d| ## Data Source The model was trained on the Universal Dependencies, curated by Google. A NER version was created by megagonlabs: https://github.com/megagonlabs/UD_Japanese-GSD Reference: Asahara, M., Kanayama, H., Tanaka, T., Miyao, Y., Uematsu, S., Mori, S., Matsumoto, Y., Omura, M., & Murawaki, Y. (2018). Universal Dependencies Version 2 for Japanese. In LREC-2018. ## Benchmarking ```bash label precision recall f1-score support DATE 0.91 0.92 0.92 206 EVENT 0.91 0.56 0.69 52 FAC 0.82 0.63 0.71 59 GPE 0.81 0.90 0.86 102 LANGUAGE 1.00 1.00 1.00 8 LAW 0.44 0.31 0.36 13 LOC 0.83 0.93 0.87 41 MONEY 0.80 1.00 0.89 20 MOVEMENT 0.38 0.55 0.44 11 NORP 0.98 0.81 0.88 57 O 0.99 0.99 0.99 11785 ORDINAL 0.79 0.94 0.86 32 ORG 0.82 0.74 0.78 179 PERCENT 1.00 1.00 1.00 16 PERSON 0.87 0.87 0.87 127 PRODUCT 0.61 0.72 0.66 50 QUANTITY 0.91 0.91 0.91 172 TIME 0.97 0.88 0.92 32 TITLE_AFFIX 0.81 0.92 0.86 24 WORK_OF_ART 0.71 0.83 0.77 48 accuracy - - 0.98 13034 macro-avg 0.82 0.82 0.81 13034 weighted-avg 0.98 0.98 0.98 13034 ``` --- layout: model title: Italian Part of Speech Tagger (from sachaarbonel) author: John Snow Labs name: bert_pos_bert_italian_cased_finetuned_pos date: 2022-05-09 tags: [bert, pos, part_of_speech, it, open_source] task: Part of Speech Tagging language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-italian-cased-finetuned-pos` is a Italian model orginally trained by `sachaarbonel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_italian_cased_finetuned_pos_it_3.4.2_3.0_1652092443943.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_italian_cased_finetuned_pos_it_3.4.2_3.0_1652092443943.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_italian_cased_finetuned_pos| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|it| |Size:|410.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sachaarbonel/bert-italian-cased-finetuned-pos - https://raw.githubusercontent.com/stefan-it/fine-tuned-berts-seq/master/scripts/preprocess.py - https://twitter.com/sachaarbonel - https://www.linkedin.com/in/sacha-arbonel --- layout: model title: Loinc Sentence Entity Resolver author: John Snow Labs name: sbiobertresolve_loinc date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted clinical NER entities to LOINC codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts LOINC Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_LOINC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_en_3.0.4_3.0_1621189494152.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_loinc_en_3.0.4_3.0_1621189494152.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') sentenceDetector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") stopwords = StopWordsCleaner.pretrained()\ .setInputCols("token")\ .setOutputCol("cleanTokens")\ .setCaseSensitive(False) word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "cleanTokens"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "cleanTokens", "ner"]) \ .setOutputCol("ner_chunk") chunk2doc = Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_loinc","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") pipeline_loinc = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, resolver]) data = spark.createDataFrame([["""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting."""]]).toDF("text") results = pipeline_loinc.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val stopwords = StopWordsCleaner.pretrained() .setInputCols("token") .setOutputCol("cleanTokens") .setCaseSensitive(False) val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "cleanTokens")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "cleanTokens", "ner")) .setOutputCol("ner_chunk") val chunk2doc = new Chunk2Doc() .setInputCols("ner_chunk") .setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols("ner_chunk_doc") .setOutputCol("sbert_embeddings") val resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_loinc","en", "clinical/models") .setInputCols(Array("sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, resolver)) val data = Seq("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.loinc").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus (T2DM), one prior episode of HTG-induced pancreatitis three years prior to presentation, associated with an acute hepatitis, and obesity with a body mass index (BMI) of 33.5 kg/m2, presented with a one-week history of polyuria, polydipsia, poor appetite, and vomiting.""") ```

## Results ```bash | | chunk | loinc_code | |---:|:--------------------------------------|:-------------| | 0 | gestational diabetes mellitus | 45636-8 | | 1 | subsequent type two diabetes mellitus | 44877-9 | | 2 | T2DM | 45636-8 | | 3 | HTG-induced pancreatitis | 66667-7 | | 4 | an acute hepatitis | 45690-5 | | 5 | obesity | 73708-0 | | 6 | a body mass index | 59574-4 | | 7 | BMI | 59574-4 | | 8 | polyuria | 28239-2 | | 9 | polydipsia | 90552-1 | | 10 | poor appetite | 28387-9 | | 11 | vomiting | 81224-8 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_loinc| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[loinc_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on standard LOINC coding system. --- layout: model title: Finnish asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot TFWav2Vec2ForCTC from aapot author: John Snow Labs name: pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot date: 2022-09-25 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot` is a Finnish model originally trained by aapot. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot_fi_4.2.0_3.0_1664094639072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot_fi_4.2.0_3.0_1664094639072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_xlsr_1b_finnish_lm_v2_by_aapot| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|3.6 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Translate Luganda to English Pipeline author: John Snow Labs name: translate_lg_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, lg, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `lg` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_lg_en_xx_2.7.0_2.4_1609689782642.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_lg_en_xx_2.7.0_2.4_1609689782642.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_lg_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_lg_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.lg.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_lg_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_model_4 TFWav2Vec2ForCTC from niclas author: John Snow Labs name: asr_model_4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_4` is a English model originally trained by niclas. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_model_4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_model_4_en_4.2.0_3.0_1664098250606.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_model_4_en_4.2.0_3.0_1664098250606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_model_4", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_model_4", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_model_4| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Italian T5ForConditionalGeneration Small Cased model (from it5) author: John Snow Labs name: t5_it5_efficient_small_el32_formal_to_informal date: 2023-01-30 tags: [it, open_source, t5, tensorflow] task: Text Generation language: it edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `it5-efficient-small-el32-formal-to-informal` is a Italian model originally trained by `it5`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_formal_to_informal_it_4.3.0_3.0_1675103238992.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_it5_efficient_small_el32_formal_to_informal_it_4.3.0_3.0_1675103238992.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_formal_to_informal","it") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_it5_efficient_small_el32_formal_to_informal","it") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_it5_efficient_small_el32_formal_to_informal| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|it| |Size:|593.5 MB| ## References - https://huggingface.co/it5/it5-efficient-small-el32-formal-to-informal - https://github.com/stefan-it - https://arxiv.org/abs/2203.03759 - https://gsarti.com - https://malvinanissim.github.io - https://arxiv.org/abs/2109.10686 - https://github.com/gsarti/it5 - https://paperswithcode.com/sota?task=Formal-to-informal+Style+Transfer&dataset=XFORMAL+%28Italian+Subset%29 --- layout: model title: German ElectraForQuestionAnswering model (from deepset) author: John Snow Labs name: electra_qa_g_base_germanquad date: 2022-06-22 tags: [de, open_source, electra, question_answering] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gelectra-base-germanquad` is a German model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_g_base_germanquad_de_4.0.0_3.0_1655921748064.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_g_base_germanquad_de_4.0.0_3.0_1655921748064.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_g_base_germanquad","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Was ist mein Name?", "Mein Name ist Clara und ich lebe in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_g_base_germanquad","de") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Was ist mein Name?", "Mein Name ist Clara und ich lebe in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.electra.base").predict("""Was ist mein Name?|||"Mein Name ist Clara und ich lebe in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_g_base_germanquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|de| |Size:|410.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deepset/gelectra-base-germanquad - https://deepset.ai/germanquad - https://deepset.ai/german-bert - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://haystack.deepset.ai/community/join --- layout: model title: ICD10CM ChunkResolver author: John Snow Labs name: chunkresolve_icd10cm_diseases_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance. ## Predicted Entities ICD10-CM Codes and their normalized definition with ``clinical_embeddings``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_diseases_clinical_en_3.0.0_3.0_1617355419289.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_diseases_clinical_en_3.0.0_3.0_1617355419289.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... icd10cmResolver = ChunkEntityResolverModel.pretrained('chunkresolve_icd10cm_diseases_clinical', 'en', "clinical/models")\ .setEnableLevenshtein(True)\ .setNeighbours(200).setAlternatives(5).setDistanceWeights([3,3,2,0,0,7])\ .setInputCols('token', 'chunk_embs_jsl')\ .setOutputCol('icd10cm_resolution') pipeline_icd10 = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, icd10cmResolver]) empty_df = spark.createDataFrame([[""]]).toDF("text") data = ["""This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret's Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU"""] pipeline_model = pipeline_icd10.fit(empty_df) light_pipeline = LightPipeline(pipeline_model) result = light_pipeline.annotate(data) ``` ```scala ... val icd10cmResolver = ChunkEntityResolverModel.pretrained('chunkresolve_icd10cm_diseases_clinical', 'en', "clinical/models") .setEnableLevenshtein(True) .setNeighbours(200).setAlternatives(5).setDistanceWeights(Array(3,3,2,0,0,7)) .setInputCols('token', 'chunk_embs_jsl') .setOutputCol('icd10cm_resolution') val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, stopwords, word_embeddings, jslNer, drugNer, jslConverter, drugConverter, jslChunkEmbeddings, drugChunkEmbeddings, icd10cmResolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret's Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash | | coords | chunk | entity | icd10cm_opts | |---|-------------|-----------------------------|-----------|-------------------------------------------------------------------------------------------| | 0 | 2::499::506 | insomnia | Diagnosis | [(G4700, Insomnia, unspecified), (G4709, Other insomnia), (F5102, Adjustment insomnia)...]| | 1 | 4::83::109 | chronic renal insufficiency | Diagnosis | [(N185, Chronic kidney disease, stage 5), (N181, Chronic kidney disease, stage 1), (N1...]| | 2 | 4::120::128 | gastritis | Diagnosis | [(K2970, Gastritis, unspecified, without bleeding), (B9681, Helicobacter pylori [H. py...]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_icd10cm_diseases_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[icd10cm]| |Language:|en| --- layout: model title: Fast Neural Machine Translation Model from Brazilian Sign Language to English author: John Snow Labs name: opus_mt_bzs_en date: 2020-12-29 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, bzs, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `bzs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bzs_en_xx_2.7.0_2.4_1609254326608.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bzs_en_xx_2.7.0_2.4_1609254326608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bzs_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Slovak to English Pipeline author: John Snow Labs name: translate_sk_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, sk, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `sk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_sk_en_xx_2.7.0_2.4_1609688872908.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_sk_en_xx_2.7.0_2.4_1609688872908.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_sk_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_sk_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.sk.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_sk_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English image_classifier_vit_animal_classifier_huggingface ViTForImageClassification from vivekRahul author: John Snow Labs name: image_classifier_vit_animal_classifier_huggingface date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_animal_classifier_huggingface` is a English model originally trained by vivekRahul. ## Predicted Entities `lion`, `tiger`, `dog`, `cat`, `elephant` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animal_classifier_huggingface_en_4.1.0_3.0_1660168778923.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_animal_classifier_huggingface_en_4.1.0_3.0_1660168778923.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_animal_classifier_huggingface", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_animal_classifier_huggingface", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_animal_classifier_huggingface| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Indemnification Relation Extraction (sm, Bidirectional) author: John Snow Labs name: legre_indemnifications date: 2022-09-28 tags: [en, legal, re, indemnification, licensed] task: Relation Extraction language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_indemnification_clause` Text Classifier to select only these paragraphs; This is a Relation Extraction model to group the different entities extracted with the Indemnification NER model (see `legner_bert_indemnifications` in Models Hub). This model is a `sm` model without meaningful directions in the relations (the model was not trained to understand if the direction of the relation is from left to right or right to left). There are bigger models in Models Hub trained also with directed relationships. ## Predicted Entities `is_indemnification_subject`, `is_indemnification_object`, `is_indemnification_indobject` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/LEGALRE_INDEMNIFICATION/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legre_indemnifications_en_1.0.0_3.0_1664361611044.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legre_indemnifications_en_1.0.0_3.0_1664361611044.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "en") \ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") tokenClassifier = legal.BertForTokenClassification.pretrained("legner_bert_indemnifications", "en", "legal/models")\ .setInputCols("token", "sentence")\ .setOutputCol("label")\ .setCaseSensitive(True) ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","label"])\ .setOutputCol("ner_chunk") # ONLY NEEDED IF YOU WANT TO FILTER RELATION PAIRS OR SYNTACTIC DISTANCE # ================= pos_tagger = nlp.PerceptronModel()\ .pretrained() \ .setInputCols(["sentence", "token"])\ .setOutputCol("pos_tags") dependency_parser = nlp.DependencyParserModel() \ .pretrained("dependency_conllu", "en") \ .setInputCols(["sentence", "pos_tags", "token"]) \ .setOutputCol("dependencies") #Set a filter on pairs of named entities which will be treated as relation candidates re_filter = legal.RENerChunksFilter()\ .setInputCols(["ner_chunk", "dependencies"])\ .setOutputCol("re_ner_chunks")\ .setMaxSyntacticDistance(20)\ .setRelationPairs(['INDEMNIFICATION_SUBJECT-INDEMNIFICATION_ACTION', 'INDEMNIFICATION_SUBJECT-INDEMNIFICATION_INDIRECT_OBJECT', 'INDEMNIFICATION_ACTION-INDEMNIFICATION', 'INDEMNIFICATION_ACTION-INDEMNIFICATION_INDIRECT_OBJECT']) # ================= reDL = legal.RelationExtractionDLModel()\ .pretrained("legre_indemnifications", "en", "legal/models")\ .setPredictionThreshold(0.5)\ .setInputCols(["re_ner_chunks", "sentence"])\ .setOutputCol("relations") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentencizer, tokenizer, tokenClassifier, ner_converter, pos_tagger, dependency_parser, re_filter, reDL]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text='''The Company shall indemnify and hold harmless HOC against any losses, claims, damages or liabilities to which it may become subject under the 1933 Act or otherwise, insofar as such losses, claims, damages or liabilities (or actions in respect thereof) arise out of or are based upon ''' data = spark.createDataFrame([[text]]).toDF("text") model = nlpPipeline.fit(data) lmodel = LightPipeline(model) res = lmodel.annotate(text) ```

## Results ```bash relation entity1 entity1_begin entity1_end chunk1 entity2 entity2_begin entity2_end chunk2 confidence 1 is_indemnification_subject INDEMNIFICATION_SUBJECT 4 10 Company INDEMNIFICATION_ACTION 32 44 hold harmless 0.8847967 2 is_indemnification_indobject INDEMNIFICATION_SUBJECT 4 10 Company INDEMNIFICATION_INDIRECT_OBJECT 46 48 HOC 0.96191925 3 is_indemnification_indobject INDEMNIFICATION_ACTION 12 26 shall indemnify INDEMNIFICATION_INDIRECT_OBJECT 46 48 HOC 0.7332646 10 is_indemnification_object INDEMNIFICATION_ACTION 32 44 hold harmless INDEMNIFICATION 70 75 claims 0.9728908 11 is_indemnification_object INDEMNIFICATION_ACTION 32 44 hold harmless INDEMNIFICATION 78 84 damages 0.9727499 12 is_indemnification_object INDEMNIFICATION_ACTION 32 44 hold harmless INDEMNIFICATION 89 99 liabilities 0.964168 ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legre_indemnifications| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.9 MB| ## References In-house annotated examples from CUAD legal dataset ## Benchmarking ```bash label Recall Precision F1 Support is_indemnification_indobject 0.966 1.000 0.982 29 is_indemnification_object 0.929 0.929 0.929 42 is_indemnification_subject 0.931 0.931 0.931 29 no_rel 0.950 0.941 0.945 100 Avg. 0.944 0.950 0.947 - Weighted-Avg. 0.945 0.945 0.945 - ``` --- layout: model title: Pipeline to Detect Clinical Entities (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_jsl_pipeline date: 2023-06-06 tags: [licensed, en, clinical, ner, ner_jsl, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_jsl](https://nlp.johnsnowlabs.com/2023/05/04/bert_token_classifier_ner_jsl_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.3.0_3.2_1686088562944.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.3.0_3.2_1686088562944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") text = """The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""" result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") val text = """The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""" val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_labels | confidence | |---:|:------------------------------------------|--------:|------:|:-----------------------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.996622 | | 1 | Caucasian | 28 | 36 | Race_Ethnicity | 0.999759 | | 2 | male | 38 | 41 | Gender | 0.999847 | | 3 | 2 days | 52 | 57 | Duration | 0.818646 | | 4 | congestion | 62 | 71 | Symptom | 0.997344 | | 5 | mom | 75 | 77 | Gender | 0.999601 | | 6 | yellow | 99 | 104 | Symptom | 0.476263 | | 7 | discharge | 106 | 114 | Symptom | 0.704853 | | 8 | nares | 135 | 139 | External_body_part_or_region | 0.999152 | | 9 | she | 147 | 149 | Gender | 0.999927 | | 10 | mild | 168 | 171 | Modifier | 0.999674 | | 11 | problems with his breathing while feeding | 173 | 213 | Symptom | 0.995353 | | 12 | perioral cyanosis | 237 | 253 | Symptom | 0.99852 | | 13 | retractions | 258 | 268 | Symptom | 0.999806 | | 14 | One day ago | 272 | 282 | RelativeDate | 0.99949 | | 15 | mom | 285 | 287 | Gender | 0.999779 | | 16 | tactile temperature | 304 | 322 | Symptom | 0.997475 | | 17 | Tylenol | 345 | 351 | Drug_BrandName | 0.998978 | | 18 | Baby-girl | 354 | 362 | Age | 0.990654 | | 19 | decreased | 382 | 390 | Symptom | 0.996808 | | 20 | intake | 397 | 402 | Symptom | 0.983608 | | 21 | His | 405 | 407 | Gender | 0.999922 | | 22 | breast-feeding | 416 | 429 | External_body_part_or_region | 0.994421 | | 23 | 20 minutes | 444 | 453 | Duration | 0.992322 | | 24 | 5 to 10 minutes | 464 | 478 | Duration | 0.969913 | | 25 | his | 493 | 495 | Gender | 0.999908 | | 26 | respiratory congestion | 497 | 518 | Symptom | 0.995677 | | 27 | He | 521 | 522 | Gender | 0.999803 | | 28 | tired | 555 | 559 | Symptom | 0.999463 | | 29 | fussy | 574 | 578 | Symptom | 0.996514 | | 30 | over the past 2 days | 580 | 599 | RelativeDate | 0.998001 | | 31 | albuterol | 642 | 650 | Drug_Ingredient | 0.99964 | | 32 | ER | 676 | 677 | Clinical_Dept | 0.998161 | | 33 | His | 680 | 682 | Gender | 0.999921 | | 34 | urine output has also decreased | 684 | 714 | Symptom | 0.971606 | | 35 | he | 726 | 727 | Gender | 0.999916 | | 36 | per 24 hours | 765 | 776 | Frequency | 0.910935 | | 37 | he | 783 | 784 | Gender | 0.999922 | | 38 | per 24 hours | 812 | 823 | Frequency | 0.921849 | | 39 | Mom | 826 | 828 | Gender | 0.999606 | | 40 | diarrhea | 841 | 848 | Symptom | 0.999849 | | 41 | His | 851 | 853 | Gender | 0.999739 | | 42 | bowel | 855 | 859 | Internal_organ_or_component | 0.999471 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_jsl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_modelo1 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-modelo1` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo1_es_4.3.0_3.0_1674218442505.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo1_es_4.3.0_3.0_1674218442505.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo1","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo1","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_modelo1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-modelo1 --- layout: model title: Named Entity Recognition in Romanian Official Documents (Large) author: John Snow Labs name: legner_romanian_official_lg date: 2022-11-10 tags: [ro, ner, legal, licensed] task: Named Entity Recognition language: ro edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a large version NER model that extracts following 14 entities from Romanian Official Documents. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE`, `DECISION`, `DECREE`, `DIRECTIVE`, `ORDINANCE`, `EMERGENCY_ORDINANCE`, `LAW`, `ORDER`, `REGULATION`, `REPORT`, `TREATY` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/LEGNER_ROMANIAN_OFFICIAL/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_lg_ro_1.0.0_3.0_1668084251147.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_lg_ro_1.0.0_3.0_1668084251147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence")\ tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_base_cased", "ro")\ .setInputCols("sentence", "token")\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_romanian_official_lg", "ro", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""LEGE nr. 159 din 25 iulie 2019 pentru modificarea și completarea Decretului-lege nr. 118 / 1990 privind acordarea unor drepturi persoanelor persecutate din motive politice de dictatura instaurată cu începere de la 6 martie 1945, precum și celor deportate în străinătate ori constituite în prizonieri și pentru modificarea Ordonanței Guvernului nr. 105 / 1999 privind acordarea unor drepturi persoanelor persecutate de către regimurile instaurate în România cu începere de la 6 septembrie 1940 până la 6 martie 1945 din motive etnice Publicat în MONITORUL OFICIAL nr. 625 din 26 iulie 2019 Parlamentul României adoptă prezenta lege. Articolul I Decretul-lege nr. 118 / 1990 privind acordarea unor drepturi persoanelor persecutate din motive politice de dictatura instaurată cu începere de la 6 martie 1945, precum și celor deportate în străinătate ori constituite în prizonieri, republicat în Monitorul Oficial al României, Partea I, nr. 631 din 23 septembrie 2009, cu modificările și completările ulterioare, se modifică și se completează după cum urmează: Articolul II Ordonanța Guvernului nr. 105 / 1999 privind acordarea unor drepturi persoanelor persecutate de către regimurile instaurate în România cu începere de la 6 septembrie 1940 până la 6 martie 1945 din motive etnice, publicată în Monitorul Oficial al României, Partea I, nr. 426 din 31 august 1999, aprobată cu modificări și completări prin Legea nr. 189 / 2000, cu modificările și completările ulterioare, se modifică după cum urmează: Această lege a fost adoptată de Parlamentul României, cu respectarea prevederilor art. 75 și ale art. 76 alin. (2) din Constituția României, republicată. PREȘEDINTELE CAMEREI DEPUTAȚILOR ION-MARCEL CIOLACU București, 25 iulie 2019."""]]).toDF("text") result = model.transform(data) ```

## Results ```bash +------------------------------------+---------+ |chunk |label | +------------------------------------+---------+ |LEGE nr. 159 din 25 iulie 2019 |LAW | |Decretului-lege nr. 118 / 1990 |DECREE | |6 martie 1945 |DATE | |Ordonanței Guvernului nr. 105 / 1999|ORDINANCE| |România |LOC | |6 septembrie 1940 |DATE | |6 martie 1945 |DATE | |26 iulie 2019 |DATE | |Parlamentul României |ORG | |Decretul-lege nr. 118 / 1990 |DECREE | |6 martie 1945 |DATE | |Monitorul Oficial al României |ORG | |23 septembrie 2009 |DATE | |Ordonanța Guvernului nr. 105 / 1999 |ORDINANCE| |România |LOC | |6 septembrie 1940 |DATE | |6 martie 1945 |DATE | |Monitorul Oficial al României |ORG | |31 august 1999 |DATE | |Legea nr. 189 / 2000 |LAW | |Parlamentul României |ORG | |Constituția României |LAW | |CAMEREI DEPUTAȚILOR |ORG | |ION-MARCEL CIOLACU |PER | |București |LOC | |25 iulie 2019 |DATE | +------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_romanian_official_lg| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.5 MB| ## References Dataset is available [here](https://zenodo.org/record/7025333#.Y2zsquxBx83). ## Benchmarking ```bash label precision recall f1-score support DATE 0.9036 0.9223 0.9128 193 DECISION 0.9831 0.9831 0.9831 59 DECREE 0.5000 1.0000 0.6667 1 DIRECTIVE 1.0000 0.6667 0.8000 3 EMERGENCY_ORDINANCE 1.0000 0.9615 0.9804 26 LAW 0.9619 0.9806 0.9712 103 LOC 0.9110 0.8365 0.8721 159 ORDER 0.9767 1.0000 0.9882 42 ORDINANCE 1.0000 0.9500 0.9744 20 ORG 0.8899 0.8879 0.8889 455 PER 0.9091 0.9821 0.9442 112 REGULATION 0.9118 0.8378 0.8732 37 REPORT 0.7778 0.7778 0.7778 9 TREATY 1.0000 1.0000 1.0000 3 micro-avg 0.9139 0.9116 0.9127 1222 macro-avg 0.9089 0.9133 0.9024 1222 weighted-avg 0.9143 0.9116 0.9124 1222 ``` --- layout: model title: Chinese Bert Embeddings (from Geotrend) author: John Snow Labs name: bert_embeddings_bert_base_zh_cased date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-zh-cased` is a Chinese model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_zh_cased_zh_3.4.2_3.0_1649669775199.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_zh_cased_zh_3.4.2_3.0_1649669775199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_zh_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|360.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-zh-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-10` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1654191695998.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10_en_4.0.0_3.0_1654191695998.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_64d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|378.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-10 --- layout: model title: English BertForQuestionAnswering model (from batterydata) author: John Snow Labs name: bert_qa_batteryscibert_uncased_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `batteryscibert-uncased-squad-v1` is a English model orginally trained by `batterydata`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_batteryscibert_uncased_squad_v1_en_4.0.0_3.0_1654179403799.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_batteryscibert_uncased_squad_v1_en_4.0.0_3.0_1654179403799.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_batteryscibert_uncased_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_batteryscibert_uncased_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_battery.scibert.uncased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_batteryscibert_uncased_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/batterydata/batteryscibert-uncased-squad-v1 - https://github.com/ShuHuang/batterybert --- layout: model title: English BertForQuestionAnswering Base Uncased model (from ksabeh) author: John Snow Labs name: bert_qa_base_uncased_attribute_correction_mlm date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-attribute-correction-mlm` is a English model originally trained by `ksabeh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_attribute_correction_mlm_en_4.0.0_3.0_1657183769786.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_attribute_correction_mlm_en_4.0.0_3.0_1657183769786.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_attribute_correction_mlm","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_attribute_correction_mlm","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_attribute_correction_mlm| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ksabeh/bert-base-uncased-attribute-correction-mlm --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from adamlin) author: John Snow Labs name: distilbert_qa_base_cased_sgd_step5000 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-sgd_qa-step5000` is a English model originally trained by `adamlin`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_sgd_step5000_en_4.3.0_3.0_1672767052489.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_sgd_step5000_en_4.3.0_3.0_1672767052489.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_sgd_step5000","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_sgd_step5000","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_sgd_step5000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/adamlin/distilbert-base-cased-sgd_qa-step5000 --- layout: model title: Part of Speech for Finnish author: John Snow Labs name: pos_ud_tdt date: 2021-03-08 tags: [part_of_speech, open_source, finnish, pos_ud_tdt, fi] task: Part of Speech Tagging language: fi edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - NOUN - ADJ - VERB - ADV - CCONJ - PUNCT - SCONJ - PRON - PROPN - AUX - ADP - NUM - X - SYM - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_tdt_fi_3.0.0_3.0_1615230305202.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_tdt_fi_3.0.0_3.0_1615230305202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_tdt", "fi") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Hei John Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_tdt", "fi") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hei John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Hei John Snow Labs! ""] token_df = nlu.load('fi.pos').predict(text) token_df ```

## Results ```bash token pos 0 Hei INTJ 1 John PROPN 2 Snow PROPN 3 Labs PROPN 4 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_tdt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|fi| --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from jgriffi) author: John Snow Labs name: xlmroberta_ner_jgriffi_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `jgriffi`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jgriffi_base_finetuned_panx_all_xx_4.1.0_3.0_1660428574153.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_jgriffi_base_finetuned_panx_all_xx_4.1.0_3.0_1660428574153.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jgriffi_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_jgriffi_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_jgriffi_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|862.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jgriffi/xlm-roberta-base-finetuned-panx-all --- layout: model title: General Oncology Pipeline author: John Snow Labs name: oncology_general_pipeline date: 2023-03-29 tags: [licensed, pipeline, oncology, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition, Assertion Status and Relation Extraction models to extract information from oncology texts. This pipeline extracts diagnoses, treatments, tests, anatomical references and demographic entities. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_general_pipeline_en_4.3.2_3.2_1680110018146.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_general_pipeline_en_4.3.2_3.2_1680110018146.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_general_pipeline", "en", "clinical/models") text = '''The patient underwent a left mastectomy for a left breast cancer two months ago. The tumor is positive for ER and PR.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_general_pipeline", "en", "clinical/models") val text = "The patient underwent a left mastectomy for a left breast cancer two months ago. The tumor is positive for ER and PR." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_general.pipeline").predict("""The patient underwent a left mastectomy for a left breast cancer two months ago. The tumor is positive for ER and PR.""") ```

## Results ```bash " ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:---------------|:-----------------| | left | Direction | | mastectomy | Cancer_Surgery | | left | Direction | | breast cancer | Cancer_Dx | | two months ago | Relative_Date | | tumor | Tumor_Finding | | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | ******************** ner_oncology_diagnosis_wip results ******************** | chunk | ner_label | |:--------------|:--------------| | breast cancer | Cancer_Dx | | tumor | Tumor_Finding | ******************** ner_oncology_tnm_wip results ******************** | chunk | ner_label | |:--------------|:------------| | breast cancer | Cancer_Dx | | tumor | Tumor | ******************** ner_oncology_therapy_wip results ******************** | chunk | ner_label | |:-----------|:---------------| | mastectomy | Cancer_Surgery | ******************** ner_oncology_test_wip results ******************** | chunk | ner_label | |:---------|:-----------------| | positive | Biomarker_Result | | ER | Biomarker | | PR | Biomarker | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:--------------|:---------------|:------------| | mastectomy | Cancer_Surgery | Past | | breast cancer | Cancer_Dx | Present | | tumor | Tumor_Finding | Present | | ER | Biomarker | Present | | PR | Biomarker | Present | ******************** re_oncology_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:--------------|:-----------------|:---------------|:--------------|:--------------| | mastectomy | Cancer_Surgery | two months ago | Relative_Date | is_related_to | | breast cancer | Cancer_Dx | two months ago | Relative_Date | is_related_to | | tumor | Tumor_Finding | ER | Biomarker | O | | tumor | Tumor_Finding | PR | Biomarker | O | | positive | Biomarker_Result | ER | Biomarker | is_related_to | | positive | Biomarker_Result | PR | Biomarker | is_related_to | ******************** re_oncology_granular_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:--------------|:-----------------|:---------------|:--------------|:--------------| | mastectomy | Cancer_Surgery | two months ago | Relative_Date | is_date_of | | breast cancer | Cancer_Dx | two months ago | Relative_Date | is_date_of | | tumor | Tumor_Finding | ER | Biomarker | O | | tumor | Tumor_Finding | PR | Biomarker | O | | positive | Biomarker_Result | ER | Biomarker | is_finding_of | | positive | Biomarker_Result | PR | Biomarker | is_finding_of | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_general_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - RelationExtractionModel --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-more_fine_tune_24465520-26265906` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906_en_4.0.0_3.0_1655985334529.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906_en_4.0.0_3.0_1655985334529.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlm_roberta.fine_tune_24465520_26265906").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_more_fine_tune_24465520_26265906| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|888.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-more_fine_tune_24465520-26265906 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_32_epochs30 TFWav2Vec2ForCTC from ying-tina author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_32_epochs30` is a English model originally trained by ying-tina. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30_en_4.2.0_3.0_1664111484072.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30_en_4.2.0_3.0_1664111484072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_32_epochs30| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from flood) author: John Snow Labs name: xlmroberta_ner_flood_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `flood`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_flood_base_finetuned_panx_de_4.1.0_3.0_1660432859459.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_flood_base_finetuned_panx_de_4.1.0_3.0_1660432859459.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_flood_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_flood_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_flood_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/flood/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Hungarian BertForQuestionAnswering model (from mcsabai) author: John Snow Labs name: bert_qa_huBert_fine_tuned_hungarian_squadv1 date: 2022-06-02 tags: [hu, open_source, question_answering, bert] task: Question Answering language: hu edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `huBert-fine-tuned-hungarian-squadv1` is a Hungarian model orginally trained by `mcsabai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_huBert_fine_tuned_hungarian_squadv1_hu_4.0.0_3.0_1654187962165.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_huBert_fine_tuned_hungarian_squadv1_hu_4.0.0_3.0_1654187962165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_huBert_fine_tuned_hungarian_squadv1","hu") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_huBert_fine_tuned_hungarian_squadv1","hu") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("hu.answer_question.squad.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_huBert_fine_tuned_hungarian_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hu| |Size:|413.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mcsabai/huBert-fine-tuned-hungarian-squadv1 --- layout: model title: English image_classifier_vit_pond_image_classification_9 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_pond_image_classification_9 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pond_image_classification_9` is a English model originally trained by SummerChiam. ## Predicted Entities `Normal`, `Boiling`, `Algae`, `NormalCement`, `NormalRain`, `BoilingNight`, `NormalNight` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_9_en_4.1.0_3.0_1660167226926.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_9_en_4.1.0_3.0_1660167226926.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_9", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_9", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_9| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal Agreements Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_agreements_bert date: 2023-03-05 tags: [en, legal, classification, clauses, agreements, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Agreements` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Agreements`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_agreements_bert_en_1.0.0_3.0_1678050691083.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_agreements_bert_en_1.0.0_3.0_1678050691083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_agreements_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Agreements]| |[Other]| |[Other]| |[Agreements]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_agreements_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Agreements 0.86 0.96 0.91 26 Other 0.97 0.90 0.93 39 accuracy - - 0.92 65 macro-avg 0.92 0.93 0.92 65 weighted-avg 0.93 0.92 0.92 65 ``` --- layout: model title: Detect biological concepts (biobert) author: John Snow Labs name: ner_bionlp_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect general biological entities like tissues, organisms, cells, etc in text using pretrained NER model. ## Predicted Entities `tissue_structure`, `Amino_acid`, `Simple_chemical`, `Organism_substance`, `Developing_anatomical_structure`, `Cell`, `Cancer`, `Cellular_component`, `Gene_or_gene_product`, `Immaterial_anatomical_entity`, `Organ`, `Organism`, `Pathological_formation`, `Organism_subdivision`, `Anatomical_system`, `Tissue` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_TUMOR/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_en_3.0.0_3.0_1617260864949.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_biobert_en_3.0.0_3.0_1617260864949.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_bionlp_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_bionlp_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.bionlp.biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bionlp_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash +-------------------------------+------+-----+------+------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +-------------------------------+------+-----+------+------+---------+------+------+ | Organ| 123.0| 59.0| 50.0| 173.0| 0.6758| 0.711| 0.693| | Pathological_formation| 82.0| 32.0| 45.0| 127.0| 0.7193|0.6457|0.6805| | Organism_substance| 75.0| 8.0| 51.0| 126.0| 0.9036|0.5952|0.7177| | tissue_structure| 412.0|162.0| 53.0| 465.0| 0.7178| 0.886|0.7931| | Cellular_component| 188.0| 46.0| 61.0| 249.0| 0.8034| 0.755|0.7785| | Tissue| 244.0| 73.0| 51.0| 295.0| 0.7697|0.8271|0.7974| | Cancer|1384.0|193.0| 144.0|1528.0| 0.8776|0.9058|0.8915| |Developing_anatomical_structure| 10.0| 3.0| 11.0| 21.0| 0.7692|0.4762|0.5882| | Immaterial_anatomical_entity| 20.0| 16.0| 21.0| 41.0| 0.5556|0.4878|0.5195| | Gene_or_gene_product|3829.0|233.0|1045.0|4874.0| 0.9426|0.7856| 0.857| | Cell|1873.0|175.0| 231.0|2104.0| 0.9146|0.8902|0.9022| | Organism| 559.0|116.0| 79.0| 638.0| 0.8281|0.8762|0.8515| | Simple_chemical| 928.0|106.0| 421.0|1349.0| 0.8975|0.6879|0.7789| +-------------------------------+------+-----+------+------+---------+------+------+ +------------------+ | macro| +------------------+ |0.6589490623994527| +------------------+ +------------------+ | micro| +------------------+ |0.8407823737981155| +------------------+ ``` --- layout: model title: Legal Transactions with affiliates Clause Binary Classifier author: John Snow Labs name: legclf_transactions_with_affiliates_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `transactions-with-affiliates` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `transactions-with-affiliates` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_transactions_with_affiliates_clause_en_1.0.0_3.2_1660123125455.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_transactions_with_affiliates_clause_en_1.0.0_3.2_1660123125455.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_transactions_with_affiliates_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[transactions-with-affiliates]| |[other]| |[other]| |[transactions-with-affiliates]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_transactions_with_affiliates_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.98 0.97 86 transactions-with-affiliates 0.92 0.88 0.90 26 accuracy - - 0.96 112 macro-avg 0.94 0.93 0.94 112 weighted-avg 0.95 0.96 0.96 112 ``` --- layout: model title: English image_classifier_vit_orcs_and_friends ViTForImageClassification from andy-0v0 author: John Snow Labs name: image_classifier_vit_orcs_and_friends date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_orcs_and_friends` is a English model originally trained by andy-0v0. ## Predicted Entities `ogre`, `goblin`, `orc`, `gremlin`, `troll` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_orcs_and_friends_en_4.1.0_3.0_1660167849859.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_orcs_and_friends_en_4.1.0_3.0_1660167849859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_orcs_and_friends", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_orcs_and_friends", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_orcs_and_friends| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English asr_wav2vec2_base_rj_try_5 TFWav2Vec2ForCTC from rjrohit author: John Snow Labs name: pipeline_asr_wav2vec2_base_rj_try_5 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_rj_try_5` is a English model originally trained by rjrohit. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_rj_try_5_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_rj_try_5_en_4.2.0_3.0_1664102652691.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_rj_try_5_en_4.2.0_3.0_1664102652691.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_rj_try_5', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_rj_try_5", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_rj_try_5| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Swahili author: John Snow Labs name: opus_mt_en_sw date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, sw, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `sw` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_sw_xx_2.7.0_2.4_1609171045871.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_sw_xx_2.7.0_2.4_1609171045871.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_sw| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish RobertaForQuestionAnswering (from PlanTL-GOB-ES) author: John Snow Labs name: roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-bne-sqac` is a Spanish model originally trained by `PlanTL-GOB-ES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac_es_4.0.0_3.0_1655736230510.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac_es_4.0.0_3.0_1655736230510.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.sqac.roberta.large.by_PlanTL-GOB-ES").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_PlanTL_GOB_ES_roberta_large_bne_sqac| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-large-bne-sqac - https://arxiv.org/abs/2107.07253 - http://www.bne.es/en/Inicio/index.html - https://arxiv.org/abs/1907.11692 - https://github.com/PlanTL-GOB-ES/lm-spanish --- layout: model title: Legal Electronic Communications Clause Binary Classifier author: John Snow Labs name: legclf_electronic_communications_clause date: 2022-12-07 tags: [en, legal, classification, clauses, electronic_communications, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `electronic-communications` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `electronic-communications`, `other` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_electronic_communications_clause_en_1.0.0_3.0_1670445124280.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_electronic_communications_clause_en_1.0.0_3.0_1670445124280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_electronic_communications_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[electronic-communications]| |[other]| |[other]| |[electronic-communications]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_electronic_communications_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support electronic-communications 1.00 1.00 1.00 25 other 1.00 1.00 1.00 73 accuracy - - 1.00 98 macro-avg 1.00 1.00 1.00 98 weighted-avg 1.00 1.00 1.00 98 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from erickfm) author: John Snow Labs name: t5_small_finetuned_bias date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-finetuned-bias` is a English model originally trained by `erickfm`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_bias_en_4.3.0_3.0_1675125947945.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_finetuned_bias_en_4.3.0_3.0_1675125947945.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_finetuned_bias","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_finetuned_bias","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_finetuned_bias| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|284.4 MB| ## References - https://huggingface.co/erickfm/t5-small-finetuned-bias - https://github.com/rpryzant/neutralizing-bias --- layout: model title: Japanese BertForQuestionAnswering Large model (from KoichiYasuoka) author: John Snow Labs name: bert_qa_large_japanese_wikipedia_ud_head date: 2022-06-28 tags: [ja, open_source, bert, question_answering] task: Question Answering language: ja edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-japanese-wikipedia-ud-head` is a Japanese model originally trained by `KoichiYasuoka`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_large_japanese_wikipedia_ud_head_ja_4.0.0_3.0_1656413609706.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_large_japanese_wikipedia_ud_head_ja_4.0.0_3.0_1656413609706.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_large_japanese_wikipedia_ud_head","ja") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_large_japanese_wikipedia_ud_head","ja") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("私の名前は何ですか？", "私の名前はクララで、私はバークレーに住んでいます。").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ja.answer_question.wikipedia.bert.large").predict("""私の名前は何ですか？|||"私の名前はクララで、私はバークレーに住んでいます。""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_large_japanese_wikipedia_ud_head| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ja| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KoichiYasuoka/bert-large-japanese-wikipedia-ud-head - https://github.com/UniversalDependencies/UD_Japanese-GSDLUW --- layout: model title: Italian DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_it_cased date: 2022-04-12 tags: [distilbert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-it-cased` is a Italian model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_it_cased_it_3.4.2_3.0_1649783835301.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_it_cased_it_3.4.2_3.0_1649783835301.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_it_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|234.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-it-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Translate Ewe to English Pipeline author: John Snow Labs name: translate_ee_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ee, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ee` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ee_en_xx_2.7.0_2.4_1609699221850.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ee_en_xx_2.7.0_2.4_1609699221850.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ee_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ee_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ee.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ee_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Chinese to English Pipeline author: John Snow Labs name: translate_zh_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, zh, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `zh` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_zh_en_xx_2.7.0_2.4_1609685820690.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_zh_en_xx_2.7.0_2.4_1609685820690.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_zh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_zh_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.zh.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_zh_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Amharic Named Entity Recognition (from mbeukman) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_finetuned_ner_amharic date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, am, open_source] task: Named Entity Recognition language: am edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-ner-amharic` is a Amharic model orginally trained by `mbeukman`. ## Predicted Entities `PER`, `ORG`, `LOC`, `DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_amharic_am_3.4.2_3.0_1652810198597.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_finetuned_ner_amharic_am_3.4.2_3.0_1652810198597.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_finetuned_ner_amharic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|am| |Size:|773.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-ner-amharic - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://www.apache.org/licenses/LICENSE-2.0 - https://github.com/Michael-Beukman/NERTransfer - h --- layout: model title: Fast Neural Machine Translation Model from English to Icelandic author: John Snow Labs name: opus_mt_en_is date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, is, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `is` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_is_xx_2.7.0_2.4_1609168104095.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_is_xx_2.7.0_2.4_1609168104095.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_is| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Embeddings Clinical author: John Snow Labs name: embeddings_clinical class: WordEmbeddingsModel language: en nav_key: models repository: clinical/models date: 2020-01-28 task: Embeddings edition: Healthcare NLP 2.4.0 spark_version: 2.4 tags: [clinical,licensed,embeddings,en] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_en_2.4.0_2.4_1580237286004.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_clinical_en_2.4.0_2.4_1580237286004.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models")\ .setInputCols("document","token")\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.glove.clinical").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | embeddings_clinical | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.4.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | en | | Dimension: | 200.0 | {:.h2_title} ## Data Source Trained on PubMed corpora https://www.nlm.nih.gov/databases/download/pubmed_medline.html --- layout: model title: Part of Speech for Basque author: John Snow Labs name: pos_ud_bdt date: 2020-07-29 23:34:00 +0800 task: Part of Speech Tagging language: eu edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [pos, eu] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bdt_eu_2.5.5_2.4_1596053577577.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bdt_eu_2.5.5_2.4_1596053577577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_bdt", "eu") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_bdt", "eu") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Iparraldeko erregea izateaz gain, mediku ingelesa eta anestesia eta higiene medikoa garatzen duen liderra da John Snow."""] pos_df = nlu.load('eu.pos').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=10, result='NOUN', metadata={'word': 'Iparraldeko'}), Row(annotatorType='pos', begin=12, end=18, result='NOUN', metadata={'word': 'erregea'}), Row(annotatorType='pos', begin=20, end=26, result='VERB', metadata={'word': 'izateaz'}), Row(annotatorType='pos', begin=28, end=31, result='NOUN', metadata={'word': 'gain'}), Row(annotatorType='pos', begin=32, end=32, result='PUNCT', metadata={'word': ','}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bdt| |Type:|pos| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|eu| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Legal Foodstuff Document Classifier (EURLEX) author: John Snow Labs name: legclf_foodstuff_bert date: 2023-03-06 tags: [en, legal, classification, clauses, foodstuff, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_foodstuff_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Foodstuff or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Foodstuff`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_foodstuff_bert_en_1.0.0_3.0_1678111793995.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_foodstuff_bert_en_1.0.0_3.0_1678111793995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_foodstuff_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Foodstuff]| |[Other]| |[Other]| |[Foodstuff]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_foodstuff_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Foodstuff 0.84 0.84 0.84 669 Other 0.81 0.80 0.80 541 accuracy - - 0.82 1210 macro-avg 0.82 0.82 0.82 1210 weighted-avg 0.82 0.82 0.82 1210 ``` --- layout: model title: Translate English to German Pipeline author: John Snow Labs name: translate_en_de date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, de, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `de` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_de_xx_2.7.0_2.4_1609687050179.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_de_xx_2.7.0_2.4_1609687050179.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_de", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_de", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.de').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_de| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Mossi to English author: John Snow Labs name: opus_mt_mos_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, mos, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `mos` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_mos_en_xx_2.7.0_2.4_1609163565820.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_mos_en_xx_2.7.0_2.4_1609163565820.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_mos_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Romanian Lemmatizer author: John Snow Labs name: lemma date: 2020-05-05 11:16:00 +0800 task: Lemmatization language: ro edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, ro] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_ro_2.5.0_2.4_1588666512149.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_ro_2.5.0_2.4_1588666512149.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "ro") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "ro") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""În afară de a fi regele nordului, John Snow este un medic englez și un lider în dezvoltarea anesteziei și igienei medicale."""] lemma_df = nlu.load('ro.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=1, result='În', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=3, end=7, result='afară', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=9, end=10, result='de', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=12, end=12, result='al', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=14, end=15, result='fi', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|ro| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_orkg_which_5e_05 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-orkg-which-5e-05` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_which_5e_05_en_4.3.0_3.0_1672766921583.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_which_5e_05_en_4.3.0_3.0_1672766921583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_which_5e_05","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_which_5e_05","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_orkg_which_5e_05| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Moussab/distilbert-base-cased-distilled-squad-orkg-which-5e-05 --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-1024-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4_en_4.0.0_3.0_1655730810758.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4_en_4.0.0_3.0_1655730810758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_1024d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_1024_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|439.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-1024-finetuned-squad-seed-4 --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in German (WikiNER 6B 300) author: John Snow Labs name: wikiner_6B_300 date: 2020-02-03 task: Named Entity Recognition language: de edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [ner, de, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 6B 300 is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_DE){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_DE.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_de_2.4.0_2.4_1579717534653.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_6B_300_de_2.4.0_2.4_1579717534653.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +--------------------------+---------+ |chunk |ner_label| +--------------------------+---------+ |William Henry Gates III |PER | |London |LOC | |US-amerikanischer |MISC | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |Chief Executive Officer |MISC | |CEO |ORG | |Mikrocomputer-Revolution |MISC | |Seattle |LOC | |Washington |LOC | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | |Personal-Computer-Software|ORG | |Gates |ORG | |CEO |ORG | |CEO |ORG | |Chief Software Architect |MISC | +--------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.4.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|de| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://de.wikipedia.org](https://de.wikipedia.org) --- layout: model title: English ALBERT Embeddings (Base, v1) author: John Snow Labs name: albert_embeddings_albert_base_v1 date: 2022-04-14 tags: [albert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-base-v1` is a English model orginally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_v1_en_3.4.2_3.0_1649954125996.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_base_v1_en_3.4.2_3.0_1649954125996.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_base_v1| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|44.8 MB| |Case sensitive:|false| ## References - https://huggingface.co/albert-base-v1 - https://arxiv.org/abs/1909.11942 - https://github.com/google-research/albert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: Explain Document Pipeline for Russian author: John Snow Labs name: explain_document_md date: 2021-03-22 tags: [open_source, russian, explain_document_md, pipeline, ru] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: ru edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_md is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_md_ru_3.0.0_3.0_1616432725151.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_md_ru_3.0.0_3.0_1616432725151.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_md', lang = 'ru') annotations = pipeline.fullAnnotate(""Здравствуйте из Джона Снежных Лабораторий! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_md", lang = "ru") val result = pipeline.fullAnnotate("Здравствуйте из Джона Снежных Лабораторий! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Здравствуйте из Джона Снежных Лабораторий! ""] result_df = nlu.load('ru.explain.md').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:------------------------------------------------|:-----------------------------------------------|:-----------------------------------------------------------|:-----------------------------------------------------------|:-------------------------------------------|:-----------------------------|:--------------------------------------|:-------------------------------| | 0 | ['Здравствуйте из Джона Снежных Лабораторий! '] | ['Здравствуйте из Джона Снежных Лабораторий!'] | ['Здравствуйте', 'из', 'Джона', 'Снежных', 'Лабораторий!'] | ['здравствовать', 'из', 'Джон', 'Снежных', 'Лабораторий!'] | ['NOUN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.0, 0.0, 0.0, 0.0,.,...]] | ['O', 'O', 'B-LOC', 'I-LOC', 'I-LOC'] | ['Джона Снежных Лабораторий!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_md| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|ru| --- layout: model title: Clinical Deidentification Pipeline (Arabic) author: John Snow Labs name: clinical_deidentification date: 2023-06-14 tags: [licensed, deidentification, clinical, ar, pipeline] task: De-identification language: ar edition: Healthcare NLP 4.4.4 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to deidentify Arabic PHI information from medical texts. The PHI information will be masked and obfuscated in the resulting text. The pipeline can mask and obfuscate CONTACT, NAME, DATE, ID, LOCATION, AGE, PATIENT, HOSPITAL, ORGANIZATION, CITY, STREET, USERNAME, SEX, IDNUM, EMAIL, ZIP, MEDICALRECORD, PROFESSION, PHONE, COUNTRY, DOCTOR, SSN, ACCOUNT, LICENSE, DLN and VIN {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_ar_4.4.4_3.0_1686734212603.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/clinical_deidentification_ar_4.4.4_3.0_1686734212603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline deid_pipeline = PretrainedPipeline("clinical_deidentification", "ar", "clinical/models") text = ''' ملاحظات سريرية - مريض الربو: التاريخ: 30 مايو 2023 اسم المريضة: ليلى حسن تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي 123456789012. العنوان: شارع المعرفة، مبنى رقم 789، حي الأمانة، جدة الرمز البريدي: 54321 البلد: المملكة العربية السعودية اسم المستشفى: مستشفى النور اسم الطبيب: د. أميرة أحمد تفاصيل الحالة: المريضة ليلى حسن، البالغة من العمر 35 عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح ''' result = deid_pipeline.annotate(text) print("\nMasked with entity labels") print("-"*30) print("\n".join(result['masked_with_entity'])) print("\nMasked with chars") print("-"*30) print("\n".join(result['masked_with_chars'])) print("\nMasked with fixed length chars") print("-"*30) print("\n".join(result['masked_fixed_length_chars'])) print("\nObfuscated") print("-"*30) print("\n".join(result['obfuscated'])) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val deid_pipeline = new PretrainedPipeline("clinical_deidentification","ar","clinical/models") val text = ''' ملاحظات سريرية - مريض الربو: التاريخ: 30 مايو 2023 اسم المريضة: ليلى حسن تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي 123456789012. العنوان: شارع المعرفة، مبنى رقم 789، حي الأمانة، جدة الرمز البريدي: 54321 البلد: المملكة العربية السعودية اسم المستشفى: مستشفى النور اسم الطبيب: د. أميرة أحمد تفاصيل الحالة: المريضة ليلى حسن، البالغة من العمر 35 عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح ''' val result = deid_pipeline.annotate(text) ```

## Results ```bash Masked with entity labels ------------------------------ ملاحظات سريرية - مريض الربو: التاريخ: [تاريخ] [تاريخ] اسم المريضة: [المريض] تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي [هاتف]. العنوان: شارع المعرفة، مبنى رقم [الرمز البريدي] حي [الموقع] الرمز البريدي: [الرمز البريدي] البلد: [المدينة] [البلد] اسم المستشفى: [الموقع] اسم الطبيب: د. [دكتور] تفاصيل الحالة: المريضة [المريض] حسن، البالغة من العمر [العمر]عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح Masked with chars ------------------------------ ملاحظات سريرية - مريض الربو: التاريخ: [٭٭٭٭٭] [٭٭] اسم المريضة: [٭٭٭٭٭٭] تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي [٭٭٭٭٭٭٭٭٭٭]. العنوان: شارع المعرفة، مبنى رقم [٭٭] حي [٭٭٭٭٭٭٭٭٭٭] الرمز البريدي: [٭٭٭] البلد: [٭٭٭٭٭٭٭٭٭٭٭٭٭] [٭٭٭٭٭٭] اسم المستشفى: [٭٭٭٭٭٭٭٭٭٭] اسم الطبيب: د. [٭٭٭٭٭٭٭٭] تفاصيل الحالة: المريضة [٭٭] حسن، البالغة من العمر ٭٭عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح Masked with fixed length chars ------------------------------ ملاحظات سريرية - مريض الربو: التاريخ: ٭٭٭٭ ٭٭٭٭ اسم المريضة: ٭٭٭٭ تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي ٭٭٭٭. العنوان: شارع المعرفة، مبنى رقم ٭٭٭٭ حي ٭٭٭٭ الرمز البريدي: ٭٭٭٭ البلد: ٭٭٭٭ ٭٭٭٭ اسم المستشفى: ٭٭٭٭ اسم الطبيب: د. ٭٭٭٭ تفاصيل الحالة: المريضة ٭٭٭٭ حسن، البالغة من العمر ٭٭٭٭عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح Obfuscated ------------------------------ ملاحظات سريرية - مريض الربو: التاريخ: 30 يونيو 2024 اسم المريضة: رياض حسيبي تم تسجيل المريض في النظام باستخدام رقم الضمان الاجتماعي 217492240818. العنوان: شارع المعرفة، مبنى رقم 616، حي شارع الحرية الرمز البريدي: 32681 البلد: تاونات موريشيوس اسم المستشفى: شارع الجزر الموريسية اسم الطبيب: د. ميرا نورة تفاصيل الحالة: المريضة ربى شحاتة حسن، البالغة من العمر 26عامًا، تعاني من مرض الربو المزمن. تشكو من ضيق التنفس والسعال المتكرر والشهيق الشديد. تم تشخيصها بمرض الربو بناءً على تاريخها الطبي واختبارات وظائف الرئة. الخطة: تم وصف مضادات الالتهاب غير الستيرويدية والموسعات القصبية لتحسين التنفس وتقليل التهيج. يجب على المريضة حمل معها جهاز الاستنشاق في حالة حدوث نوبة ربو حادة. يتعين على المريضة تجنب التحسس من العوامل المسببة للربو، مثل الدخان والغبار والحيوانات الأليفة. يجب مراقبة وظائف الرئة بانتظام ومتابعة التعليمات الطبية المتعلقة بمرض الربو. تعليم المريضة بشأن كيفية استخدام جهاز الاستنشاق بشكل صحيح وتقنيات التنفس الصحيح ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|clinical_deidentification| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|ar| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ContextualParserModel - ChunkMergeModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - DeIdentificationModel - Finisher --- layout: model title: Fast Neural Machine Translation Model from Basque (Family) to English author: John Snow Labs name: opus_mt_euq_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, euq, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `euq` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_euq_en_xx_2.7.0_2.4_1609164976603.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_euq_en_xx_2.7.0_2.4_1609164976603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_euq_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Stopwords Remover for Kannada language (82 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, kn, open_source] task: Stop Words Removal language: kn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_kn_3.4.1_3.0_1646672976164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_kn_3.4.1_3.0_1646672976164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------+ |result | +--------------+ |[ಉತ್ತಮವಾಗಿಲ್ಲ]| +--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|kn| |Size:|1.8 KB| --- layout: model title: English LongformerForQuestionAnswering model (Squad dataset) author: John Snow Labs name: longformer_qa_base_4096_finetuned_squadv2 date: 2022-06-26 tags: [en, open_source, longformer, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: LongformerForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `longformer-base-4096-finetuned-squadv2` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/longformer_qa_base_4096_finetuned_squadv2_en_4.0.0_3.0_1656255377204.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/longformer_qa_base_4096_finetuned_squadv2_en_4.0.0_3.0_1656255377204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = LongformerForQuestionAnswering.pretrained("longformer_qa_base_4096_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = LongformerForQuestionAnswering.pretrained("longformer_qa_base_4096_finetuned_squadv2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.longformer.base_v2").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|longformer_qa_base_4096_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|551.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/longformer-base-4096-finetuned-squadv2 - https://rajpurkar.github.io/SQuAD-explorer/ - https://arxiv.org/abs/2004.05150 --- layout: model title: English DistilBertForQuestionAnswering model (from andi611) author: John Snow Labs name: distilbert_qa_base_uncased_qa_with_ner date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-qa-with-ner` is a English model originally trained by `andi611`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_qa_with_ner_en_4.0.0_3.0_1654727168244.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_qa_with_ner_en_4.0.0_3.0_1654727168244.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_qa_with_ner","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_qa_with_ner","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.conll.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_qa_with_ner| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/andi611/distilbert-base-uncased-qa-with-ner --- layout: model title: Word2Vec Embeddings in Walloon (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, wa, open_source] task: Embeddings language: wa edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_wa_3.4.1_3.0_1647466983817.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_wa_3.4.1_3.0_1647466983817.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python # YOUR NER HERE # ... embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") chunk_converter = nlp.ChunkConverter() \ .setInputCols(["entity"]) \ .setOutputCol("ner_chunk") assertion = legal.AssertionDLModel.pretrained("legassertion_time", "en", "legal/models")\ .setInputCols(["sentence", "ner_chunk", "embeddings"]) \ .setOutputCol("assertion") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, tokenizer, embeddings, ner, chunk_converter, assertion ]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) lp = LightPipeline(model) texts = ["The subsidiaries of Atlantic Inc will participate in a merging operation", "The Conditions and Warranties of this agreement might be modified"] lp.annotate(texts) ```

## Results ```bash chunk,begin,end,entity_type,assertion Atlantic Inc,20,31,ORG,FUTURE chunk,begin,end,entity_type,assertion Conditions and Warranties,4,28,DOC,POSSIBLE ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legassertion_time| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, doc_chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|2.2 MB| ## References In-house annotations on financial and legal corpora ## Benchmarking ```bash label tp fp fn prec rec f1 PRESENT 201 11 16 0.9481132 0.9262672 0.937063 POSSIBLE 171 3 6 0.9827586 0.9661017 0.974359 FUTURE 119 6 4 0.952 0.9674796 0.959677 PAST 270 16 10 0.9440559 0.9642857 0.954063 Macro-average 761 36 36 0.9567319 0.9560336 0.9563826 Micro-average 761 36 36 0.9548306 0.9548306 0.9548306 ``` --- layout: model title: ICD10 to ICD9 Code Mapping author: John Snow Labs name: icd10_icd9_mapping date: 2021-12-22 tags: [icd10, icd9, en, clinical, licensed, code_mapping] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps ICD10 codes to ICD9 codes without using any text data. You’ll just feed a comma or white space-delimited ICD10 codes and it will return the corresponding ICD9 codes as a list. {:.btn-box} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.1.Healthcare_Code_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10_icd9_mapping_en_3.3.4_2.4_1640175449509.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10_icd9_mapping_en_3.3.4_2.4_1640175449509.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd10_icd9_mapping", "en", "clinical/models") pipeline.annotate('E669 R630 J988') ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10_icd9_mapping", "en", "clinical/models") val result = pipeline.annotate('E669 R630 J988') ``` {:.nlu-block} ```python import nlu nlu.load("en.icd10_icd9.mapping").predict("""E669 R630 J988""") ```

## Results ```bash {'document': ['E669 R630 J988'], 'icd10': ['E669', 'R630', 'J988'], 'icd9': ['27800', '7830', '5198']} Note: | ICD10 | Details | | ---------- | ----------------------------:| | E669 | Obesity | | R630 | Anorexia | | J988 | Other specified respiratory disorders | | ICD9 | Details | | ---------- | ---------------------------:| | 27800 | Obesity | | 7830 | Anorexia | | 5198 | Other diseases of respiratory system | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10_icd9_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|545.2 KB| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel --- layout: model title: English DistilBertForQuestionAnswering Cased model (from LucasS) author: John Snow Labs name: distilbert_qa_absa date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilBertABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_absa_en_4.3.0_3.0_1672766295348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_absa_en_4.3.0_3.0_1672766295348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_absa","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_absa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_absa| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LucasS/distilBertABSA --- layout: model title: Finnish XlmRoBertaForQuestionAnswering (from Gantenbein) author: John Snow Labs name: xlm_roberta_qa_ADDI_FI_XLM_R date: 2022-06-23 tags: [fi, open_source, question_answering, xlmroberta] task: Question Answering language: fi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ADDI-FI-XLM-R` is a Finnish model originally trained by `Gantenbein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_ADDI_FI_XLM_R_fi_4.0.0_3.0_1655983152909.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_ADDI_FI_XLM_R_fi_4.0.0_3.0_1655983152909.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_ADDI_FI_XLM_R","fi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_ADDI_FI_XLM_R","fi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("fi.answer_question.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_ADDI_FI_XLM_R| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|fi| |Size:|777.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Gantenbein/ADDI-FI-XLM-R --- layout: model title: English image_classifier_vit_shirt_identifier ViTForImageClassification from b25mayank3 author: John Snow Labs name: image_classifier_vit_shirt_identifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_shirt_identifier` is a English model originally trained by b25mayank3. ## Predicted Entities `Big Check shirt`, `Formal Shirt`, `casual shirt`, `denim shirt` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_shirt_identifier_en_4.1.0_3.0_1660171643901.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_shirt_identifier_en_4.1.0_3.0_1660171643901.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_shirt_identifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_shirt_identifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_shirt_identifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Word2Vec Embeddings in Serbian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, sr, open_source] task: Embeddings language: sr edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sr_3.4.1_3.0_1647456008397.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_sr_3.4.1_3.0_1647456008397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_no_label_1e_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_deepset_base_squad2_orkg_no_label_1e_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_deepset_base_squad2_orkg_no_label_1e_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Moussab/deepset-roberta-base-squad2-orkg-no-label-1e-4 --- layout: model title: English RobertaForSequenceClassification Cased model (from liamcripwell) author: John Snow Labs name: roberta_classifier_ctrl44_clf date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification, tensorflow] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ctrl44-clf` is a English model originally trained by `liamcripwell`. ## Predicted Entities `rephrase`, `ignore`, `syntax-split`, `discourse-split` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_ctrl44_clf_en_4.2.4_3.0_1670624790107.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_ctrl44_clf_en_4.2.4_3.0_1670624790107.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_ctrl44_clf","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_ctrl44_clf","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_ctrl44_clf| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|467.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/liamcripwell/ctrl44-clf --- layout: model title: English Legal RoBERTa Embeddings (CaseLaw, Base, Cased) author: John Snow Labs name: roberta_embeddings_legal_roberta_base date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `legal-roberta-base` is a English model orginally trained by `saibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_legal_roberta_base_en_3.4.2_3.0_1649946738367.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_legal_roberta_base_en_3.4.2_3.0_1649946738367.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_legal_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|468.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/saibo/legal-roberta-base - https://www.kaggle.com/uspto/patent-litigations - https://case.law/ - https://www.kaggle.com/bigquery/patents - https://www.kaggle.com/sohier/beyond-queries-exploring-the-bigquery-api --- layout: model title: Vietnamese asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese TFWav2Vec2ForCTC from leduytan93 author: John Snow Labs name: asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese date: 2022-09-26 tags: [wav2vec2, vi, audio, open_source, asr] task: Automatic Speech Recognition language: vi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese` is a Vietnamese model originally trained by leduytan93. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese_vi_4.2.0_3.0_1664197522821.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese_vi_4.2.0_3.0_1664197522821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese", "vi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese", "vi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Fine_Tune_XLSR_Wav2Vec2_Speech2Text_Vietnamese| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|vi| |Size:|1.2 GB| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_32_epochs30 TFWav2Vec2ForCTC from ying-tina author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_32_epochs30 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_32_epochs30` is a English model originally trained by ying-tina. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_32_epochs30_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_32_epochs30_en_4.2.0_3.0_1664111278307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_32_epochs30_en_4.2.0_3.0_1664111278307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_32_epochs30", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_32_epochs30", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_32_epochs30| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|354.9 MB| --- layout: model title: Pipeline to Detect Cellular/Molecular Biology Entities author: John Snow Labs name: bert_token_classifier_ner_jnlpba_cellular_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_jnlpba_cellular](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_jnlpba_cellular_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jnlpba_cellular_pipeline_en_4.3.0_3.2_1679303520732.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jnlpba_cellular_pipeline_en_4.3.0_3.2_1679303520732.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_jnlpba_cellular_pipeline", "en", "clinical/models") text = '''The results suggest that activation of protein kinase C, but not new protein synthesis, is required for IL-2 induction of IFN-gamma and GM-CSF cytoplasmic mRNA. It also was observed that suppression of cytokine gene expression by these agents was independent of the inhibition of proliferation. These data indicate that IL-2 and IL-12 may have distinct signaling pathways leading to the induction of IFN-gammaand GM-CSFgene expression, andthatthe NK3.3 cell line may serve as a novel model for dissecting the biochemical and molecular events involved in these pathways. A functional T-cell receptor signaling pathway is required for p95vav activity. Stimulation of the T-cell antigen receptor ( TCR ) induces activation of multiple tyrosine kinases, resulting in phosphorylation of numerous intracellular substrates. One substrate is p95vav, which is expressed exclusively in hematopoietic and trophoblast cells..''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_jnlpba_cellular_pipeline", "en", "clinical/models") val text = "The results suggest that activation of protein kinase C, but not new protein synthesis, is required for IL-2 induction of IFN-gamma and GM-CSF cytoplasmic mRNA. It also was observed that suppression of cytokine gene expression by these agents was independent of the inhibition of proliferation. These data indicate that IL-2 and IL-12 may have distinct signaling pathways leading to the induction of IFN-gammaand GM-CSFgene expression, andthatthe NK3.3 cell line may serve as a novel model for dissecting the biochemical and molecular events involved in these pathways. A functional T-cell receptor signaling pathway is required for p95vav activity. Stimulation of the T-cell antigen receptor ( TCR ) induces activation of multiple tyrosine kinases, resulting in phosphorylation of numerous intracellular substrates. One substrate is p95vav, which is expressed exclusively in hematopoietic and trophoblast cells.." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------------------------|--------:|------:|:------------|-------------:| | 0 | protein kinase C | 39 | 54 | protein | 0.993263 | | 1 | IL-2 | 104 | 107 | protein | 0.969095 | | 2 | IFN-gamma and GM-CSF cytoplasmic mRNA | 122 | 158 | RNA | 0.998495 | | 3 | cytokine gene | 202 | 214 | DNA | 0.953537 | | 4 | IL-2 | 320 | 323 | protein | 0.999317 | | 5 | IL-12 | 329 | 333 | protein | 0.999216 | | 6 | IFN-gammaand GM-CSFgene | 400 | 422 | protein | 0.995236 | | 7 | NK3.3 cell line | 447 | 461 | cell_line | 0.998958 | | 8 | T-cell receptor | 583 | 597 | protein | 0.987655 | | 9 | p95vav | 633 | 638 | protein | 0.999857 | | 10 | T-cell antigen receptor | 669 | 691 | protein | 0.99891 | | 11 | TCR | 695 | 697 | protein | 0.998049 | | 12 | tyrosine kinases | 732 | 747 | protein | 0.999636 | | 13 | p95vav | 834 | 839 | protein | 0.999842 | | 14 | hematopoietic and trophoblast cells | 876 | 910 | cell_type | 0.999709 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_jnlpba_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Detect Radiology Concepts (WIP) author: John Snow Labs name: jsl_rd_ner_wip_greedy_clinical date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract clinical entities from Radiology reports using pretrained NER model. ## Predicted Entities `Kidney_Disease`, `HDL`, `Diet`, `Test`, `Imaging_Technique`, `Triglycerides`, `Obesity`, `Duration`, `Weight`, `Social_History_Header`, `ImagingTest`, `Labour_Delivery`, `Disease_Syndrome_Disorder`, `Communicable_Disease`, `Overweight`, `Units`, `Smoking`, `Score`, `Substance_Quantity`, `Form`, `Race_Ethnicity`, `Modifier`, `Hyperlipidemia`, `ImagingFindings`, `Psychological_Condition`, `OtherFindings`, `Cerebrovascular_Disease`, `Date`, `Test_Result`, `VS_Finding`, `Employment`, `Death_Entity`, `Gender`, `Oncological`, `Heart_Disease`, `Medical_Device`, `Total_Cholesterol`, `ManualFix`, `Time`, `Route`, `Pulse`, `Admission_Discharge`, `RelativeDate`, `O2_Saturation`, `Frequency`, `RelativeTime`, `Hypertension`, `Alcohol`, `Allergen`, `Fetus_NewBorn`, `Birth_Entity`, `Age`, `Respiration`, `Medical_History_Header`, `Oxygen_Therapy`, `Section_Header`, `LDL`, `Treatment`, `Vital_Signs_Header`, `Direction`, `BMI`, `Pregnancy`, `Sexually_Active_or_Sexual_Orientation`, `Symptom`, `Clinical_Dept`, `Measurements`, `Height`, `Family_History_Header`, `Substance`, `Strength`, `Injury_or_Poisoning`, `Relationship_Status`, `Blood_Pressure`, `Drug`, `Temperature`, `EKG_Findings`, `Diabetes`, `BodyPart`, `Vaccine`, `Procedure`, `Dosage` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_clinical_en_3.0.0_3.0_1617260438155.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_rd_ner_wip_greedy_clinical_en_3.0.0_3.0_1617260438155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("jsl_rd_ner_wip_greedy_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("jsl_rd_ner_wip_greedy_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl.wip.clinical.rd").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_rd_ner_wip_greedy_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash entity tp fp fn total precision recall f1 VS_Finding 306.0 129.0 119.0 425.0 0.7034 0.72 0.7116 Direction 8717.0 678.0 616.0 9333.0 0.9278 0.934 0.9309 Respiration 224.0 28.0 18.0 242.0 0.8889 0.9256 0.9069 Cerebrovascular_Disease 149.0 57.0 64.0 213.0 0.7233 0.6995 0.7112 Family_History_Header 315.0 1.0 3.0 318.0 0.9968 0.9906 0.9937 Heart_Disease 1087.0 198.0 141.0 1228.0 0.8459 0.8852 0.8651 ImagingFindings 5568.0 1112.0 1627.0 7195.0 0.8335 0.7739 0.8026 RelativeTime 422.0 138.0 100.0 522.0 0.7536 0.8084 0.78 Strength 96.0 51.0 54.0 150.0 0.6531 0.64 0.6465 BodyPart 20155.0 1698.0 1860.0 22015.0 0.9223 0.9155 0.9189 Smoking 151.0 16.0 5.0 156.0 0.9042 0.9679 0.935 Medical_Device 8162.0 885.0 821.0 8983.0 0.9022 0.9086 0.9054 EKG_Findings 131.0 37.0 83.0 214.0 0.7798 0.6121 0.6859 Pulse 382.0 44.0 50.0 432.0 0.8967 0.8843 0.8904 Psychological_Condition 195.0 32.0 43.0 238.0 0.859 0.8193 0.8387 Triglycerides 18.0 0.0 0.0 18.0 1.0 1.0 1.0 Overweight 6.0 2.0 1.0 7.0 0.75 0.8571 0.8 Obesity 68.0 3.0 5.0 73.0 0.9577 0.9315 0.9444 Admission_Discharge 376.0 26.0 24.0 400.0 0.9353 0.94 0.9377 HDL 11.0 0.0 5.0 16.0 1.0 0.6875 0.8148 Diabetes 227.0 9.0 12.0 239.0 0.9619 0.9498 0.9558 Section_Header 13630.0 476.0 413.0 14043.0 0.9663 0.9706 0.9684 Age 1174.0 129.0 94.0 1268.0 0.901 0.9259 0.9133 O2_Saturation 122.0 34.0 29.0 151.0 0.7821 0.8079 0.7948 Drug 9391.0 1505.0 928.0 10319.0 0.8619 0.9101 0.8853 Kidney_Disease 296.0 28.0 53.0 349.0 0.9136 0.8481 0.8796 Test 3980.0 721.0 925.0 4905.0 0.8466 0.8114 0.8286 Communicable_Disease 40.0 18.0 12.0 52.0 0.6897 0.7692 0.7273 Hypertension 163.0 16.0 10.0 173.0 0.9106 0.9422 0.9261 Oxygen_Therapy 123.0 36.0 27.0 150.0 0.7736 0.82 0.7961 Test_Result 1607.0 374.0 458.0 2065.0 0.8112 0.7782 0.7944 Modifier 1229.0 435.0 593.0 1822.0 0.7386 0.6745 0.7051 BMI 21.0 4.0 7.0 28.0 0.84 0.75 0.7925 Labour_Delivery 117.0 38.0 62.0 179.0 0.7548 0.6536 0.7006 Employment 414.0 65.0 93.0 507.0 0.8643 0.8166 0.8398 Fetus_NewBorn 118.0 68.0 87.0 205.0 0.6344 0.5756 0.6036 Clinical_Dept 1937.0 189.0 133.0 2070.0 0.9111 0.9357 0.9233 Time 637.0 43.0 27.0 664.0 0.9368 0.9593 0.9479 Procedure 7578.0 953.0 1088.0 8666.0 0.8883 0.8745 0.8813 ImagingTest 1712.0 213.0 281.0 1993.0 0.8894 0.859 0.8739 Diet 79.0 44.0 82.0 161.0 0.6423 0.4907 0.5563 Oncological 1088.0 188.0 103.0 1191.0 0.8527 0.9135 0.882 LDL 20.0 7.0 1.0 21.0 0.7407 0.9524 0.8333 Symptom 15940.0 3662.0 3035.0 18975.0 0.8132 0.8401 0.8264 Temperature 240.0 28.0 25.0 265.0 0.8955 0.9057 0.9006 Vital_Signs_Header 850.0 34.0 52.0 902.0 0.9615 0.9424 0.9518 Total_Cholesterol 43.0 6.0 7.0 50.0 0.8776 0.86 0.8687 Relationship_Status 51.0 3.0 9.0 60.0 0.9444 0.85 0.8947 Blood_Pressure 353.0 18.0 117.0 470.0 0.9515 0.7511 0.8395 Injury_or_Poisoning 1003.0 311.0 241.0 1244.0 0.7633 0.8063 0.7842 Treatment 335.0 98.0 91.0 426.0 0.7737 0.7864 0.78 Pregnancy 214.0 99.0 86.0 300.0 0.6837 0.7133 0.6982 Vaccine 29.0 3.0 10.0 39.0 0.9063 0.7436 0.8169 Height 105.0 10.0 45.0 150.0 0.913 0.7 0.7925 Disease_Syndrome_Disorder 8466.0 1568.0 1533.0 9999.0 0.8437 0.8467 0.8452 Frequency 1263.0 237.0 173.0 1436.0 0.842 0.8795 0.8604 Route 219.0 35.0 144.0 363.0 0.8622 0.6033 0.7099 Duration 978.0 199.0 338.0 1316.0 0.8309 0.7432 0.7846 Death_Entity 35.0 17.0 16.0 51.0 0.6731 0.6863 0.6796 Alcohol 102.0 24.0 21.0 123.0 0.8095 0.8293 0.8193 Date 840.0 43.0 13.0 853.0 0.9513 0.9848 0.9677 Hyperlipidemia 44.0 4.0 1.0 45.0 0.9167 0.9778 0.9462 Social_History_Header 284.0 6.0 27.0 311.0 0.9793 0.9132 0.9451 ManualFix 50.0 2.0 7.0 57.0 0.9615 0.8772 0.9174 Imaging_Technique 845.0 240.0 98.0 943.0 0.7788 0.8961 0.8333 Race_Ethnicity 141.0 0.0 5.0 146.0 1.0 0.9658 0.9826 RelativeDate 1691.0 394.0 194.0 1885.0 0.811 0.8971 0.8519 Gender 6800.0 105.0 130.0 6930.0 0.9848 0.9812 0.983 Dosage 122.0 67.0 81.0 203.0 0.6455 0.601 0.6224 Medical_History_Header 486.0 10.0 19.0 505.0 0.9798 0.9624 0.971 Sexually_Active_or_Sexual_Orientation 12.0 0.0 5.0 17.0 1.0 0.7059 0.8276 Substance 102.0 11.0 22.0 124.0 0.9027 0.8226 0.8608 Weight 346.0 26.0 65.0 411.0 0.9301 0.8418 0.8838 macro - - - - - - 0.8038 micro - - - - - - 0.8793 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from shwetha) author: John Snow Labs name: distilbert_qa_shwetha_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `shwetha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_shwetha_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772737063.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_shwetha_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772737063.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shwetha_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_shwetha_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_shwetha_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/shwetha/distilbert-base-uncased-finetuned-squad --- layout: model title: Drug Side Effect Classifier (BioBERT) author: John Snow Labs name: bert_sequence_classifier_vop_drug_side_effect date: 2023-06-13 tags: [licensed, classification, side_effect, drug, vop, en, tensorflow] task: Text Classification language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT based](https://github.com/dmis-lab/biobert) classifier that can classify informal texts (such as tweets or forum posts) according to the presence of drug side effects. ## Predicted Entities `Drug_AE`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_drug_side_effect_en_4.4.3_3.0_1686668996540.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_vop_drug_side_effect_en_4.4.3_3.0_1686668996540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------+---------+ |text |result | +--------------------------------------------------------------+---------+ |I felt kind of dizzy after taking that medication for a month.|[Drug_AE]| |I took antibiotics last week and everything went well. |[Other] | +--------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_vop_drug_side_effect| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I’m 20 year old girl. I’m diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I’m taking weekly supplement of vitamin D and 1000 mcg b12 daily. I’m taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I’m facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label precision recall f1-score support Drug_AE 0.861111 0.788136 0.823009 118 Other 0.875622 0.921466 0.897959 191 accuracy - - 0.870550 309 macro_avg 0.868367 0.854801 0.860484 309 weighted_avg 0.870081 0.870550 0.869337 309 ``` --- layout: model title: Legal No third party beneficiaries Clause Binary Classifier author: John Snow Labs name: legclf_no_third_party_beneficiaries_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-third-party-beneficiaries` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-third-party-beneficiaries` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_third_party_beneficiaries_clause_en_1.0.0_3.2_1660122706229.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_third_party_beneficiaries_clause_en_1.0.0_3.2_1660122706229.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_third_party_beneficiaries_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-third-party-beneficiaries]| |[other]| |[other]| |[no-third-party-beneficiaries]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_third_party_beneficiaries_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-third-party-beneficiaries 0.96 0.96 0.96 49 other 0.98 0.98 0.98 130 accuracy - - 0.98 179 macro-avg 0.97 0.97 0.97 179 weighted-avg 0.98 0.98 0.98 179 ``` --- layout: model title: English T5ForConditionalGeneration Cased model (from jeremyccollinsmpi) author: John Snow Labs name: t5_autotrain_inference_probability_3_900329401 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-inference_probability_3-900329401` is a English model originally trained by `jeremyccollinsmpi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_autotrain_inference_probability_3_900329401_en_4.3.0_3.0_1675099900080.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_autotrain_inference_probability_3_900329401_en_4.3.0_3.0_1675099900080.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_autotrain_inference_probability_3_900329401","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_autotrain_inference_probability_3_900329401","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_autotrain_inference_probability_3_900329401| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|846.1 MB| ## References - https://huggingface.co/jeremyccollinsmpi/autotrain-inference_probability_3-900329401 --- layout: model title: Part of Speech for Turkish author: John Snow Labs name: pos_ud_imst date: 2020-05-03 12:43:00 +0800 task: Part of Speech Tagging language: tr edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, tr] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_imst_tr_2.5.0_2.4_1587480006078.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_imst_tr_2.5.0_2.4_1587480006078.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_imst", "tr") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("John Snow, kuzeyin kralı olmanın yanı sıra bir İngiliz doktordur ve anestezi ve tıbbi hijyenin geliştirilmesinde liderdir.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_imst", "tr") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("John Snow, kuzeyin kralı olmanın yanı sıra bir İngiliz doktordur ve anestezi ve tıbbi hijyenin geliştirilmesinde liderdir.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""John Snow, kuzeyin kralı olmanın yanı sıra bir İngiliz doktordur ve anestezi ve tıbbi hijyenin geliştirilmesinde liderdir."""] pos_df = nlu.load('tr.pos.ud_imst').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=3, result='NOUN', metadata={'word': 'John'}), Row(annotatorType='pos', begin=5, end=8, result='PROPN', metadata={'word': 'Snow'}), Row(annotatorType='pos', begin=9, end=9, result='PUNCT', metadata={'word': ','}), Row(annotatorType='pos', begin=11, end=17, result='NOUN', metadata={'word': 'kuzeyin'}), Row(annotatorType='pos', begin=19, end=23, result='NOUN', metadata={'word': 'kralı'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_imst| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|tr| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Pipeline to Detect Clinical Entities (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_jsl_pipeline date: 2023-06-13 tags: [ner_jsl, ner, berfortokenclassification, en, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_jsl](https://nlp.johnsnowlabs.com/2022/03/21/bert_token_classifier_ner_jsl_en_2_4.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.4.4_3.2_1686664116632.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_jsl_pipeline_en_4.4.4_3.2_1686664116632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_jsl_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_token_ner_jsl.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby-girl also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash Results | | ner_chunk | begin | end | ner_label | confidence | |---:|:---------------------------------|--------:|------:|:-------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 0.999456 | | 1 | Caucasian male | 28 | 41 | Demographics | 0.9901 | | 2 | congestion | 62 | 71 | Symptom | 0.997918 | | 3 | mom | 75 | 77 | Demographics | 0.999013 | | 4 | yellow discharge | 99 | 114 | Symptom | 0.998663 | | 5 | nares | 135 | 139 | Body_Part | 0.998609 | | 6 | she | 147 | 149 | Demographics | 0.999442 | | 7 | mild problems with his breathing | 168 | 199 | Symptom | 0.930385 | | 8 | perioral cyanosis | 237 | 253 | Symptom | 0.99819 | | 9 | retractions | 258 | 268 | Symptom | 0.999783 | | 10 | One day ago | 272 | 282 | Date_Time | 0.999386 | | 11 | mom | 285 | 287 | Demographics | 0.999835 | | 12 | tactile temperature | 304 | 322 | Symptom | 0.999352 | | 13 | Tylenol | 345 | 351 | Drug | 0.999762 | | 14 | Baby-girl | 354 | 362 | Age | 0.980529 | | 15 | decreased p.o. intake | 382 | 402 | Symptom | 0.998978 | | 16 | His | 405 | 407 | Demographics | 0.999913 | | 17 | breast-feeding | 416 | 429 | Body_Part | 0.99954 | | 18 | his | 493 | 495 | Demographics | 0.999661 | | 19 | respiratory congestion | 497 | 518 | Symptom | 0.834984 | | 20 | He | 521 | 522 | Demographics | 0.999858 | | 21 | tired | 555 | 559 | Symptom | 0.999516 | | 22 | fussy | 574 | 578 | Symptom | 0.997592 | | 23 | over the past 2 days | 580 | 599 | Date_Time | 0.994786 | | 24 | albuterol | 642 | 650 | Drug | 0.999735 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_jsl_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: BioBERT Embeddings (Clinical) author: John Snow Labs name: biobert_clinical_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for generic clinical text. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_clinical_base_cased_en_2.6.0_2.4_1598343387227.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_clinical_base_cased_en_2.6.0_2.4_1598343387227.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = BertEmbeddings.pretrained("biobert_clinical_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer']], ["text"])) ``` ```scala val embeddings = BertEmbeddings.pretrained("biobert_clinical_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I hate cancer").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer"] embeddings_df = nlu.load('en.embed.biobert.clinical_base_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_biobert_clinical_base_cased_embeddings I [0.2206662893295288, 0.41324421763420105, -0.3... hate [-0.19311018288135529, 0.6037888526916504, -0.... cancer [0.2895708680152893, 0.22499887645244598, -0.5... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_clinical_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465514 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465514` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465514_en_4.0.0_3.0_1655985998052.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465514_en_4.0.0_3.0_1655985998052.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465514","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465514","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465514.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465514| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465514 --- layout: model title: Embeddings BioVec author: John Snow Labs name: embeddings_biovec class: WordEmbeddingsModel language: en nav_key: models repository: clinical/models date: 2020-06-02 task: Embeddings edition: Healthcare NLP 2.5.0 spark_version: 2.4 tags: [clinical,embeddings,en] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/embeddings_biovec_en_2.5.0_2.4_1591068211397.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/embeddings_biovec_en_2.5.0_2.4_1591068211397.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python model = WordEmbeddingsModel.pretrained("embeddings_biovec","en","clinical/models")\ .setInputCols("document","token")\ .setOutputCol("word_embeddings") ``` ```scala val model = WordEmbeddingsModel.pretrained("embeddings_biovec","en","clinical/models") .setInputCols("document","token") .setOutputCol("word_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.glove.biovec").predict("""Put your text here.""") ```

{:.h2_title} ## Results Word2Vec feature vectors based on ``embeddings_biovec``. {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | embeddings_biovec | | Type: | WordEmbeddingsModel | | Compatibility: | Spark NLP 2.5.0+ | | License: | Licensed | | Edition: | Official | |Input labels: | [document, token] | |Output labels: | [word_embeddings] | | Language: | en | | Dimension: | 300.0 | {:.h2_title} ## Data Source Trained on PubMed corpora https://github.com/ncbi-nlp/BioSentVec --- layout: model title: Extract Clinical Entities from Voice of the Patient Documents (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_emb_clinical_medium date: 2023-06-06 tags: [licensed, clinical, en, ner, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `Gender`, `Employment`, `BodyPart`, `Vaccine`, `Age`, `PsychologicalCondition`, `Form`, `Drug`, `Substance`, `ClinicalDept`, `Laterality`, `DateTime`, `Test`, `Route`, `Disease`, `AdmissionDischarge`, `Dosage`, `Duration`, `VitalTest`, `Frequency`, `Symptom`, `Allergen`, `Procedure`, `RelationshipStatus`, `HealthStatus`, `Modifier`, `TestResult`, `InjuryOrPoisoning`, `SubstanceQuantity`, `MedicalDevice`, `Treatment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_medium_en_4.4.3_3.0_1686073490433.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_emb_clinical_medium_en_4.4.3_3.0_1686073490433.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_gaa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_gaa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.gaa.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_gaa_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: BERT LaBSE Sentence Embeddings author: John Snow Labs name: labse date: 2020-09-23 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, xx] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language-agnostic BERT sentence embedding model supporting 109 languages: The language-agnostic BERT sentence embedding encodes text into high dimensional vectors. The model is trained and optimized to produce similar representations exclusively for bilingual sentence pairs that are translations of each other. So it can be used for mining for translations of a sentence in a larger corpus. The details are described in the paper "[Language-agnostic BERT Sentence Embedding. July 2020](https://arxiv.org/abs/2007.01852)" {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/labse_xx_2.6.0_2.4_1600858075633.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/labse_xx_2.6.0_2.4_1600858075633.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("labse", "xx") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP', 'Many thanks']], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("labse", "xx") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I love NLP", "Many thanks").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP", "Many thanks"] embeddings_df = nlu.load('xx.embed_sentence.labse').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence xx_embed_sentence_labse_embeddings 0 I love NLP [-0.060951583087444305, -0.011645414866507053,... 1 Many thanks [0.002173778833821416, -0.05513454228639603, 0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|labse| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[xx]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/LaBSE/1](https://tfhub.dev/google/LaBSE/1) --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-0` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0_en_4.0.0_3.0_1654191670990.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0_en_4.0.0_3.0_1654191670990.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_64d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_64_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|378.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-64-finetuned-squad-seed-0 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_12_h_512 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-12_H-512` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_512_zh_4.2.4_3.0_1670021556439.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_12_h_512_zh_4.2.4_3.0_1670021556439.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_512","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_12_h_512","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_12_h_512| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|184.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-12_H-512 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: English BertForQuestionAnswering model (from ncduy) author: John Snow Labs name: bert_qa_bert_base_cased_finetuned_squad_test date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-cased-finetuned-squad-test` is a English model orginally trained by `ncduy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_cased_finetuned_squad_test_en_4.0.0_3.0_1654179794462.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_cased_finetuned_squad_test_en_4.0.0_3.0_1654179794462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_cased_finetuned_squad_test","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_cased_finetuned_squad_test","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_cased.by_ncduy").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_cased_finetuned_squad_test| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ncduy/bert-base-cased-finetuned-squad-test --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from ZYW) author: John Snow Labs name: distilbert_qa_squad_zyw_model date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-en-de-es-vi-zh-model` is a Multilingual model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_zyw_model_xx_4.3.0_3.0_1672775639500.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_zyw_model_xx_4.3.0_3.0_1672775639500.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_zyw_model","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_zyw_model","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_zyw_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-en-de-es-vi-zh-model --- layout: model title: Legal Assignment Clause Binary Classifier author: John Snow Labs name: legclf_assignment_clause date: 2023-02-13 tags: [en, legal, classification, assignment, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `assignment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `assignment`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_assignment_clause_en_1.0.0_3.0_1676302845555.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_assignment_clause_en_1.0.0_3.0_1676302845555.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_assignment_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[assignment]| |[other]| |[other]| |[assignment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_assignment_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support assignment 0.95 1.00 0.98 20 other 1.00 0.91 0.95 11 accuracy - - 0.97 31 macro-avg 0.98 0.95 0.96 31 weighted-avg 0.97 0.97 0.97 31 ``` --- layout: model title: BioBERT Sentence Embeddings (Discharge) author: John Snow Labs name: sent_biobert_discharge_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for discharge summaries. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_discharge_base_cased_en_2.6.0_2.4_1598349530721.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_discharge_base_cased_en_2.6.0_2.4_1598349530721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_discharge_base_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_discharge_base_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.biobert.discharge_base_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_discharge_base_cased_embeddings I hate cancer [0.3155321180820465, 0.37484583258628845, -0.4... Antibiotics arent painkiller [0.3543206453323364, 0.0787968561053276, -0.08... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_discharge_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: Hindi XlmRoBertaForQuestionAnswering (from abhishek) author: John Snow Labs name: xlm_roberta_qa_autonlp_hindi_question_answering_23865268 date: 2022-06-23 tags: [hi, open_source, question_answering, xlmroberta] task: Question Answering language: hi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-hindi-question-answering-23865268` is a Hindi model originally trained by `abhishek`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_hindi_question_answering_23865268_hi_4.0.0_3.0_1655984202768.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_hindi_question_answering_23865268_hi_4.0.0_3.0_1655984202768.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_hindi_question_answering_23865268","hi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_hindi_question_answering_23865268","hi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("hi.answer_question.xlm_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_hindi_question_answering_23865268| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|hi| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/abhishek/autonlp-hindi-question-answering-23865268 --- layout: model title: Pipeline to Detect Drug Chemicals author: John Snow Labs name: ner_drugs_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, drug, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_drugs](https://nlp.johnsnowlabs.com/2021/03/31/ner_drugs_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_drugs_pipeline_en_3.4.1_3.0_1647872661131.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_drugs_pipeline_en_3.4.1_3.0_1647872661131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_drugs_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.") ``` ```scala val pipeline = new PretrainedPipeline("ner_drugs_pipeline", "en", "clinical/models") pipeline.annotate("The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.drugs.pipeline").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.BACKGROUND: At present, it is one of the most important issues for the treatment of breast cancer to develop the standard therapy for patients previously treated with anthracyclines and taxanes. With the objective of determining the usefulnessof vinorelbine monotherapy in patients with advanced or recurrent breast cancerafter standard therapy, we evaluated the efficacy and safety of vinorelbine inpatients previously treated with anthracyclines and taxanes.""") ```

## Results ```bash +-----------------+---------+ |chunk |ner | +-----------------+---------+ |potassium |DrugChem | |anthracyclines |DrugChem | |taxanes |DrugChem | |vinorelbine |DrugChem | |vinorelbine |DrugChem | |anthracyclines |DrugChem | |taxanes |DrugChem | +-----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_drugs_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_bahasa_indonesia TFWav2Vec2ForCTC from Bagus author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_bahasa_indonesia` is a Modern Greek (1453-) model originally trained by Bagus. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia_el_4.2.0_3.0_1664108133476.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia_el_4.2.0_3.0_1664108133476.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_bahasa_indonesia| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_el4 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-el4` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_el4_en_4.3.0_3.0_1675116178104.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_el4_en_4.3.0_3.0_1675116178104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_el4","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_el4","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_el4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|995.5 MB| ## References - https://huggingface.co/google/t5-efficient-large-el4 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_ff6000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-ff6000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff6000_en_4.3.0_3.0_1675123549632.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff6000_en_4.3.0_3.0_1675123549632.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_ff6000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_ff6000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_ff6000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|86.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-ff6000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Translate English to Oromo Pipeline author: John Snow Labs name: translate_en_om date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, om, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `om` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_om_xx_2.7.0_2.4_1609690425970.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_om_xx_2.7.0_2.4_1609690425970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_om", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_om", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.om').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_om| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ioanfr) author: John Snow Labs name: distilbert_qa_ioanfr_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `ioanfr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ioanfr_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771445817.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ioanfr_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771445817.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ioanfr_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ioanfr_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ioanfr_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ioanfr/distilbert-base-uncased-finetuned-squad --- layout: model title: ELECTRA Sentence Embeddings(ELECTRA Large) author: John Snow Labs name: sent_electra_large_uncased date: 2020-08-27 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description ELECTRA is a BERT-like model that is pre-trained as a discriminator in a set-up resembling a generative adversarial network (GAN). It was originally published by: Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning: [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/forum?id=r1xMH1BtvB), ICLR 2020. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_electra_large_uncased_en_2.6.0_2.4_1598489955147.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_electra_large_uncased_en_2.6.0_2.4_1598489955147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_electra_large_uncased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_electra_large_uncased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.electra_large_uncased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_electra_large_uncased_embeddings I hate cancer [-0.5168956518173218, -0.4284093976020813, -0.... Antibiotics aren't painkiller [0.03924501687288284, 0.28086787462234497, 0.3... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_electra_large_uncased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/google/electra_large/2 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab0_by_tahazakir TFWav2Vec2ForCTC from tahazakir author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab0_by_tahazakir` is a English model originally trained by tahazakir. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir_en_4.2.0_3.0_1664037102995.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir_en_4.2.0_3.0_1664037102995.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab0_by_tahazakir| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Classifier for Adverse Drug Events author: John Snow Labs name: classifierdl_ade_biobert date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.7.1 spark_version: 2.4 tags: [licensed, clinical, en, classifier] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify text/sentence in two categories: - `True` : The sentence is talking about a possible ADE - `False` : The sentences doesn’t have any information about an ADE. ## Predicted Entities `True`, `False` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_biobert_en_2.7.1_2.4_1611243410222.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_biobert_en_2.7.1_2.4_1611243410222.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained('biobert_pubmed_base_cased')\ .setInputCols(["document", 'token'])\ .setOutputCol("word_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["document", "word_embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifier = ClassifierDLModel.pretrained('classifierdl_ade_biobert', 'en', 'clinical/models')\ .setInputCols(['document', 'token', 'sentence_embeddings'])\ .setOutputCol('class') nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, embeddings, sentence_embeddings, classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate(["I feel a bit drowsy & have a little blurred vision after taking an insulin", "I feel great after taking tylenol"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("document", "token")) .setOutputCol("word_embeddings") val sentence_embeddings = new SentenceEmbeddings() .setInputCols(Array("document", "word_embeddings")) .setOutputCol("sentence_embeddings") .setPoolingStrategy("AVERAGE") val classifier = ClassifierDLModel.pretrained("classifierdl_ade_biobert", "en", "clinical/models") .setInputCols(Array("document", "token", "sentence_embeddings")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, embeddings, sentence_embeddings, classifier)) val data = Seq("""I feel a bit drowsy & have a little blurred vision after taking an insulin, I feel great after taking tylenol""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.ade.biobert").predict("""I feel a bit drowsy & have a little blurred vision after taking an insulin""") ```

## Results ```bash | | text | label | |--:|:---------------------------------------------------------------------------|:------| | 0 | I feel a bit drowsy & have a little blurred vision after taking an insulin | True | | 1 | I feel great after taking tylenol | False | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_ade_biobert| |Compatibility:|Healthcare NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|biobert_pubmed_base_cased| ## Data Source Trained on a custom dataset comprising of CADEC, DRUG-AE and Twimed. ## Benchmarking ```bash label precision recall f1-score support False 0.96 0.94 0.95 6923 True 0.71 0.79 0.75 1359 micro-avg 0.91 0.91 0.91 8282 macro-avg 0.83 0.86 0.85 8282 weighted-avg 0.92 0.91 0.91 8282 ``` --- layout: model title: Legal Authority Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_authority_bert date: 2023-03-05 tags: [en, legal, classification, clauses, authority, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Authority` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Authority`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_authority_bert_en_1.0.0_3.0_1678050593552.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_authority_bert_en_1.0.0_3.0_1678050593552.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_authority_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Authority]| |[Other]| |[Other]| |[Authority]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_authority_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Authority 0.93 0.93 0.93 56 Other 0.95 0.95 0.95 82 accuracy - - 0.94 138 macro-avg 0.94 0.94 0.94 138 weighted-avg 0.94 0.94 0.94 138 ``` --- layout: model title: Classifier for Genders - SBERT author: John Snow Labs name: classifierdl_gender_sbert date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.7.1 spark_version: 2.4 tags: [en, licensed, classifier, clinical] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model classifies the gender of the patient in the clinical document using context. ## Predicted Entities `Female`, `Male`, `Unknown` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CLASSIFICATION_GENDER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/21_Gender_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_sbert_en_2.7.1_2.4_1611248306976.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_gender_sbert_en_2.7.1_2.4_1611248306976.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Female ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_gender_sbert| |Compatibility:|Spark NLP 2.7.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|sbiobert_base_cased_mli| ## Data Source This model is trained on more than four thousands clinical documents (radiology reports, pathology reports, clinical visits etc.), annotated internally. ## Benchmarking ```bash precision recall f1-score support Female 0.9390 0.9747 0.9565 237 Male 0.9561 0.8720 0.9121 125 Unknown 0.8491 0.8824 0.8654 51 accuracy 0.9322 413 macro avg 0.9147 0.9097 0.9113 413 weighted avg 0.9331 0.9322 0.9318 413 ``` --- layout: model title: Twitter XLM-RoBERTa Base (twitter_xlm_roberta_base) author: John Snow Labs name: twitter_xlm_roberta_base date: 2021-05-25 tags: [xx, multilingual, embeddings, xlm_roberta, open_source] task: Embeddings language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description [XLM-RoBERTa](https://ai.facebook.com/blog/-xlm-r-state-of-the-art-cross-lingual-understanding-through-self-supervision/) is a scaled cross-lingual sentence encoder. It is trained on 2.5T of data across 100 languages data filtered from Common Crawl. XLM-R achieves state-of-the-arts results on multiple cross-lingual benchmarks. The XLM-RoBERTa model was proposed in [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau, Kartikay Khandelwal, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco GuzmÃ¡n, Edouard Grave, Myle Ott, Luke Zettlemoyer, and Veselin Stoyanov. It is based on Facebook's RoBERTa model released in 2019. It is a large multi-lingual language model, trained on 2.5TB of filtered CommonCrawl data. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/twitter_xlm_roberta_base_xx_3.1.0_2.4_1621962368880.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/twitter_xlm_roberta_base_xx_3.1.0_2.4_1621962368880.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = XlmRoBertaEmbeddings.pretrained("twitter_xlm_roberta_base", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") ``` ```scala val embeddings = XlmRoBertaEmbeddings.pretrained("twitter_xlm_roberta_base", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("xx.embed.xlm.twitter").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|twitter_xlm_roberta_base| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|xx| |Case sensitive:|true| ## Data Source - [https://huggingface.co/cardiffnlp/twitter-xlm-roberta-base](https://huggingface.co/cardiffnlp/twitter-xlm-roberta-base) - [https://github.com/cardiffnlp/xlm-t](https://github.com/cardiffnlp/xlm-t) --- layout: model title: Arabic Bert Embeddings (from Kamel) author: John Snow Labs name: bert_embeddings_DarijaBERT date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `DarijaBERT` is a Arabic model orginally trained by `Kamel`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_DarijaBERT_ar_3.4.2_3.0_1649677857720.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_DarijaBERT_ar_3.4.2_3.0_1649677857720.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_DarijaBERT| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|554.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/Kamel/DarijaBERT - https://github.com/AIOXLABS/DBert --- layout: model title: Word2Vec Embeddings in Polish (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, pl, open_source] task: Embeddings language: pl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pl_3.4.1_3.0_1647451641210.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pl_3.4.1_3.0_1647451641210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_512_A_8_cord19_200616_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_512_A_8_cord19_200616_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid_cord19.bert.uncased_4l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_512_A_8_cord19_200616_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|107.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-512_A-8_cord19-200616_squad2_covid-qna --- layout: model title: English Bert Embeddings (from GroNLP) author: John Snow Labs name: bert_embeddings_hateBERT date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `hateBERT` is a English model orginally trained by `GroNLP`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_hateBERT_en_3.4.2_3.0_1649671840480.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_hateBERT_en_3.4.2_3.0_1649671840480.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_hateBERT| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/GroNLP/hateBERT - https://www.semanticscholar.org/author/Tommaso-Caselli/1864635 - https://www.semanticscholar.org/author/Valerio-Basile/3101511 - https://www.semanticscholar.org/author/Jelena-Mitrovic/145157863 - https://www.semanticscholar.org/author/M.-Granitzer/2389675 - https://aclanthology.org/2021.woah-1.3/ - https://osf.io/tbd58/?view_onlycb79b3228d4248ddb875eb1803525ad8 --- layout: model title: French CamemBert Embeddings (from linyi) author: John Snow Labs name: camembert_embeddings_linyi_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `linyi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_linyi_generic_model_fr_3.4.4_3.0_1653989548358.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_linyi_generic_model_fr_3.4.4_3.0_1653989548358.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_linyi_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/linyi/dummy-model --- layout: model title: English image_classifier_vit_finetuned_eurosat_kornia ViTForImageClassification from nielsr author: John Snow Labs name: image_classifier_vit_finetuned_eurosat_kornia date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_finetuned_eurosat_kornia` is a English model originally trained by nielsr. ## Predicted Entities `Residential`, `AnnualCrop`, `Highway`, `Pasture`, `SeaLake`, `Industrial`, `HerbaceousVegetation`, `River`, `PermanentCrop`, `Forest` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_finetuned_eurosat_kornia_en_4.1.0_3.0_1660165618186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_finetuned_eurosat_kornia_en_4.1.0_3.0_1660165618186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_finetuned_eurosat_kornia", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_finetuned_eurosat_kornia", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_finetuned_eurosat_kornia| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering model (from healx) author: John Snow Labs name: bert_qa_biomedical_slot_filling_reader_large date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `biomedical-slot-filling-reader-large` is a English model orginally trained by `healx`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_biomedical_slot_filling_reader_large_en_4.0.0_3.0_1654185806879.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_biomedical_slot_filling_reader_large_en_4.0.0_3.0_1654185806879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_biomedical_slot_filling_reader_large","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_biomedical_slot_filling_reader_large","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bio_medical.bert.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_biomedical_slot_filling_reader_large| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/healx/biomedical-slot-filling-reader-large - https://arxiv.org/abs/2109.08564 --- layout: model title: English image_classifier_vit_pond_image_classification_11 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_pond_image_classification_11 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pond_image_classification_11` is a English model originally trained by SummerChiam. ## Predicted Entities `NormalCement0`, `Boiling0`, `NormalNight0`, `Algae0`, `BoilingNight0`, `NormalRain0`, `Normal0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_11_en_4.1.0_3.0_1660168212591.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_11_en_4.1.0_3.0_1660168212591.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_11", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_11", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_11| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Fast Neural Machine Translation Model from English to Semitic Languages author: John Snow Labs name: opus_mt_en_sem date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, sem, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `sem` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_sem_xx_2.7.0_2.4_1609167506537.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_sem_xx_2.7.0_2.4_1609167506537.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_sem| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: SDOH Under Treatment For Classification author: John Snow Labs name: genericclassifier_sdoh_under_treatment_sbiobert_cased_mli date: 2023-05-30 tags: [en, licensed, sdoh, generic_classifier, under_treatment, biobert] task: Text Classification language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting if the patient is under treatment or not. If under treatment is not mentioned in the text, it is regarded as “Not_Under_Treatment_Or_Not_Mentioned”. The model is trained by using GenericClassifierApproach annotator. `Under_Treatment`: The patient is under treatment. `Not_Under_Treatment_Or_Not_Mentioned`: The patient is not under treatment or it is not mentioned in the clinical notes. ## Predicted Entities `Under_Treatment`, `Not_Under_Treatment_Or_Not_Mentioned` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_CLASSIFICATION_GENERIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/SOCIAL_DETERMINANT_CLASSIFICATION.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_under_treatment_sbiobert_cased_mli_en_4.4.2_3.0_1685475327309.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_under_treatment_sbiobert_cased_mli_en_4.4.2_3.0_1685475327309.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+--------------------------------------+ | text| result| +----------------------------------------------------------------------------------------------------+--------------------------------------+ |Patient is a 50-year-old male who was diagnosed with hepatitis C. He has received a treatment pla...| [Under_Treatment]| |Patient has been living with chronic migraines for several years. She has not pursued any specifi...|[Not_Under_Treatment_Or_Not_Mentioned]| |Sarah, a 55-year-old woman with a history of high cholesterol and a family history of heart disea...| [Under_Treatment]| |John, a 60-year-old man with a history of smoking and high blood pressure, presented to his prima...|[Not_Under_Treatment_Or_Not_Mentioned]| +----------------------------------------------------------------------------------------------------+--------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_under_treatment_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH Project ## Benchmarking ```bash label precision recall f1-score support Not_Under_Treatment_Or_Not_Mentioned 0.88 0.75 0.81 281 Under_Treatment 0.86 0.93 0.90 458 accuracy - - 0.86 739 macro-avg 0.87 0.84 0.85 739 weighted-avg 0.87 0.86 0.86 739 ``` --- layout: model title: NER Model Finder author: John Snow Labs name: ner_model_finder date: 2022-01-21 tags: [pretrainedpipeline, clinical, ner, en, licensed] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.3.2 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is trained with bert embeddings and can be used to find the most appropriate NER model given the entity name. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_MODEL_FINDER/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_model_finder_en_3.3.2_2.4_1642758002888.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_model_finder_en_3.3.2_2.4_1642758002888.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_pipeline = PretrainedPipeline("ner_model_finder", "en", "clinical/models") result = ner_pipeline.annotate("medication") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_pipeline = PretrainedPipeline("ner_model_finder","en","clinical/models") val result = ner_pipeline.annotate("medication") ``` {:.nlu-block} ```python import nlu nlu.load("en.ner.model_finder.pipeline").predict("""Put your text here.""") ```

## Results ```bash {'model_names': ["['ner_posology', 'ner_posology_large', 'ner_posology_small', 'ner_posology_greedy', 'ner_drugs_large', 'ner_posology_experimental', 'ner_drugs_greedy', 'ner_ade_clinical', 'ner_jsl_slim', 'ner_posology_healthcare', 'ner_ade_healthcare', 'jsl_ner_wip_modifier_clinical', 'ner_ade_clinical', 'ner_jsl_greedy', 'ner_risk_factors']"]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_model_finder| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|155.8 MB| ## Included Models - DocumentAssembler - BertSentenceEmbeddings - SentenceEntityResolverModel - Finisher --- layout: model title: Word2Vec Embeddings in Chinese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_zh_3.4.1_3.0_1647290907204.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_zh_3.4.1_3.0_1647290907204.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bat_en| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual XLMRobertaForTokenClassification Cased model (from opensource) author: John Snow Labs name: xlmroberta_ner_extract_names date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `extract_names` is a Multilingual model originally trained by `opensource`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_extract_names_xx_4.1.0_3.0_1660422227464.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_extract_names_xx_4.1.0_3.0_1660422227464.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_extract_names","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_extract_names","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_extract_names| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|967.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/opensource/extract_names --- layout: model title: Fast Neural Machine Translation Model from Macedonian to English author: John Snow Labs name: opus_mt_mk_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, mk, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `mk` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_mk_en_xx_2.7.0_2.4_1609165073677.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_mk_en_xx_2.7.0_2.4_1609165073677.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_mk_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from Mr-Wick) author: John Snow Labs name: roberta_qa_Roberta date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Roberta` is a English model originally trained by `Mr-Wick`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_Roberta_en_4.0.0_3.0_1655727223761.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_Roberta_en_4.0.0_3.0_1655727223761.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_Roberta","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_Roberta","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.by_Mr-Wick").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_Roberta| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|461.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Mr-Wick/Roberta --- layout: model title: Pipeline to Mapping ICD10-CM Codes with Their Corresponding UMLS Codes author: John Snow Labs name: icd10cm_umls_mapping date: 2022-06-27 tags: [icd10cm, umls, pipeline, chunk_mapper, clinical, licensed, en] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `icd10cm_umls_mapper` model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_3.5.3_3.0_1656366054366.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_3.5.3_3.0_1656366054366.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd10cm_umls_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(["M8950", "R822", "R0901"]) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10cm_umls_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(Array("M8950", "R822", "R0901")) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd10cm.umls").predict("""Put your text here.""") ```

## Results ```bash | | icd10cm_code | umls_code | |---:|:---------------|:------------| | 0 | M8950 | C4721411 | | 1 | R822 | C0159076 | | 2 | R0901 | C0004044 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|952.4 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Catalan, Valencian asr_Wav2Vec2_Large_XLSR_53_catalan TFWav2Vec2ForCTC from PereLluis13 author: John Snow Labs name: asr_Wav2Vec2_Large_XLSR_53_catalan date: 2022-09-24 tags: [wav2vec2, ca, audio, open_source, asr] task: Automatic Speech Recognition language: ca edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_Large_XLSR_53_catalan` is a Catalan, Valencian model originally trained by PereLluis13. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Wav2Vec2_Large_XLSR_53_catalan_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_Large_XLSR_53_catalan_ca_4.2.0_3.0_1664036578113.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_Large_XLSR_53_catalan_ca_4.2.0_3.0_1664036578113.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Wav2Vec2_Large_XLSR_53_catalan", "ca")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Wav2Vec2_Large_XLSR_53_catalan", "ca") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Wav2Vec2_Large_XLSR_53_catalan| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ca| |Size:|1.2 GB| --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from Cole) author: John Snow Labs name: xlmroberta_ner_cole_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `Cole`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cole_base_finetuned_panx_de_4.1.0_3.0_1660429422413.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cole_base_finetuned_panx_de_4.1.0_3.0_1660429422413.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cole_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cole_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_cole_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Cole/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal Independent Contractor Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_independent_contractor_agreement_bert date: 2023-01-26 tags: [en, legal, classification, contractor, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_independent_contractor_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `independent-contractor-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `independent-contractor-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_independent_contractor_agreement_bert_en_1.0.0_3.0_1674732769102.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_independent_contractor_agreement_bert_en_1.0.0_3.0_1674732769102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_independent_contractor_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[independent-contractor-agreement]| |[other]| |[other]| |[independent-contractor-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_independent_contractor_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support independent-contractor-agreement 0.98 0.96 0.97 67 other 0.97 0.99 0.98 116 accuracy - - 0.98 183 macro-avg 0.98 0.97 0.98 183 weighted-avg 0.98 0.98 0.98 183 ``` --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_news_pretrain_roberta_FT_new_newsqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `news_pretrain_roberta_FT_new_newsqa` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_news_pretrain_roberta_FT_new_newsqa_en_4.0.0_3.0_1655729181858.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_news_pretrain_roberta_FT_new_newsqa_en_4.0.0_3.0_1655729181858.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_news_pretrain_roberta_FT_new_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_news_pretrain_roberta_FT_new_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.roberta.qa_ft_new.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_news_pretrain_roberta_FT_new_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/news_pretrain_roberta_FT_new_newsqa --- layout: model title: Polish DistilBertForTokenClassification Cased model (from clarin-pl) author: John Snow Labs name: distilbert_token_classifier_fastpdn_distiluse date: 2023-03-14 tags: [pl, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: pl edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `FastPDN-distiluse` is a Polish model originally trained by `clarin-pl`. ## Predicted Entities `nam_num_phone`, `nam_loc_land_mountain`, `nam_loc_gpe_subdivision`, `nam_adj_country`, `nam_oth_currency`, `nam_loc_country_region`, `nam_pro_title_document`, `nam_loc_gpe_admin1`, `nam_eve_human_sport`, `nam_loc_hydronym_sea`, `nam_fac_park`, `nam_adj_city`, `nam_loc_land_region`, `nam_liv_animal`, `nam_liv_person`, `nam_pro_media_tv`, `nam_fac_bridge`, `nam_pro_model_car`, `nam_oth_tech`, `nam_oth_position`, `nam_loc_land_island`, `nam_liv_habitant`, `nam_pro_award`, `nam_pro_title_article`, `nam_org_group`, `nam_num_house`, `nam_pro_title_book`, `nam_pro_media_periodic`, `nam_pro_media_web`, `nam_pro_title_treaty`, `nam_loc_gpe_conurbation`, `nam_pro_software_game`, `nam_pro_brand`, `nam_fac_goe`, `nam_loc_historical_region`, `nam_pro`, `nam_pro_media_radio`, `nam_pro_title`, `nam_loc_hydronym_river`, `nam_loc_land`, `nam_org_group_team`, `nam_fac_system`, `nam_org_company`, `nam_pro_title_song`, `nam_loc_land_peak`, `nam_eve`, `nam_loc_hydronym_ocean`, `nam_org_group_band`, `nam_liv_character`, `nam_loc_gpe_admin2`, `nam_org_organization`, `nam_adj_person`, `nam_eve_human`, `nam_org_nation`, `nam_loc_gpe_district`, `nam_liv_god`, `nam_org_political_party`, `nam_oth_data_format`, `nam_loc_land_continent`, `nam_fac_goe_stop`, `nam_loc`, `nam_oth`, `nam_loc_gpe_admin3`, `nam_pro_media`, `nam_loc_gpe_city`, `nam_loc_hydronym_lake`, `nam_pro_title_tv`, `nam_oth_license`, `nam_org_organization_sub`, `nam_adj`, `nam_loc_hydronym`, `nam_oth_www`, `nam_org_institution`, `nam_pro_vehicle`, `nam_pro_software`, `nam_loc_gpe_country`, `nam_eve_human_cultural`, `nam_fac_road`, `nam_pro_title_album`, `nam_loc_astronomical`, `nam_eve_human_holiday`, `nam_fac_square` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_fastpdn_distiluse_pl_4.3.1_3.0_1678783020131.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_fastpdn_distiluse_pl_4.3.1_3.0_1678783020131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_fastpdn_distiluse","pl") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_fastpdn_distiluse","pl") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_fastpdn_distiluse| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|pl| |Size:|509.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/clarin-pl/FastPDN-distiluse - https://gitlab.clarin-pl.eu/information-extraction/poldeepner2 - https://gitlab.clarin-pl.eu/grupa-wieszcz/ner/fast-pdn - https://clarin-pl.eu/dspace/bitstream/handle/11321/294/WytyczneKPWr-jednostkiidentyfikacyjne.pdf --- layout: model title: English BertForQuestionAnswering Cased model (from motiondew) author: John Snow Labs name: bert_qa_set_date_1_lr_3e_5_bs_32_ep_3 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-set_date_1-lr-3e-5-bs-32-ep-3` is a English model originally trained by `motiondew`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_lr_3e_5_bs_32_ep_3_en_4.0.0_3.0_1657188352417.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_set_date_1_lr_3e_5_bs_32_ep_3_en_4.0.0_3.0_1657188352417.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_lr_3e_5_bs_32_ep_3","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_set_date_1_lr_3e_5_bs_32_ep_3","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_set_date_1_lr_3e_5_bs_32_ep_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/motiondew/bert-set_date_1-lr-3e-5-bs-32-ep-3 --- layout: model title: English DistilBertForQuestionAnswering model (from MYX4567) author: John Snow Labs name: distilbert_qa_MYX4567_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `MYX4567`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_MYX4567_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724267012.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_MYX4567_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724267012.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_MYX4567_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_MYX4567_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_MYX4567_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/MYX4567/distilbert-base-uncased-finetuned-squad --- layout: model title: BioBERT Sentence Embeddings (Clinical) author: John Snow Labs name: sent_biobert_clinical_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for generic clinical text. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_clinical_base_cased_en_2.6.0_2.4_1598349343675.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_clinical_base_cased_en_2.6.0_2.4_1598349343675.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_clinical_base_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_clinical_base_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.biobert.clinical_base_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_clinical_base_cased_embeddings I hate cancer [0.397987425327301, 0.6472950577735901, -0.551... Antibiotics aren't painkiller [0.19467104971408844, 0.3496762812137604, -0.3... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_clinical_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: Swedish BertForMaskedLM Base Cased model (from KB) author: John Snow Labs name: bert_embeddings_kb_base_swedish_cased date: 2022-12-02 tags: [sv, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: sv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-swedish-cased` is a Swedish model originally trained by `KB`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_kb_base_swedish_cased_sv_4.2.4_3.0_1670018989680.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_kb_base_swedish_cased_sv_4.2.4_3.0_1670018989680.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kb_base_swedish_cased","sv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kb_base_swedish_cased","sv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_kb_base_swedish_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|sv| |Size:|468.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/KB/bert-base-swedish-cased --- layout: model title: Translate English to Hebrew Pipeline author: John Snow Labs name: translate_en_he date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, he, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `he` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_he_xx_2.7.0_2.4_1609687340750.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_he_xx_2.7.0_2.4_1609687340750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_he", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_he", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.he').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_he| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Judgments Clause Binary Classifier author: John Snow Labs name: legclf_judgments_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `judgments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `judgments` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_judgments_clause_en_1.0.0_3.2_1660122586859.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_judgments_clause_en_1.0.0_3.2_1660122586859.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_judgments_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[judgments]| |[other]| |[other]| |[judgments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_judgments_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support judgments 1.00 0.91 0.96 35 other 0.98 1.00 0.99 120 accuracy - - 0.98 155 macro-avg 0.99 0.96 0.97 155 weighted-avg 0.98 0.98 0.98 155 ``` --- layout: model title: Explain Document pipeline for Portuguese (explain_document_lg) author: John Snow Labs name: explain_document_lg date: 2021-03-23 tags: [open_source, portuguese, explain_document_lg, pipeline, pt] supported: true task: [Named Entity Recognition, Lemmatization] language: pt edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_lg is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_lg_pt_3.0.0_3.0_1616505297906.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_lg_pt_3.0.0_3.0_1616505297906.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_lg', lang = 'pt') annotations = pipeline.fullAnnotate(""Olá de John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_lg", lang = "pt") val result = pipeline.fullAnnotate("Olá de John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Olá de John Snow Labs! ""] result_df = nlu.load('pt.explain.lg').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:----------------------------|:---------------------------|:---------------------------------------|:---------------------------------------|:--------------------------------------------|:-----------------------------|:--------------------------------------|:--------------------| | 0 | ['Olá de John Snow Labs! '] | ['Olá de John Snow Labs!'] | ['Olá', 'de', 'John', 'Snow', 'Labs!'] | ['Olá', 'de', 'John', 'Snow', 'Labs!'] | ['PROPN', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.4388400018215179,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'I-PER'] | ['John Snow Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|pt| --- layout: model title: Fast Neural Machine Translation Model from American Sign Language to Swedish author: John Snow Labs name: opus_mt_ase_sv date: 2021-06-01 tags: [open_source, seq2seq, translation, ase, sv, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ase target languages: sv {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ase_sv_xx_3.1.0_2.4_1622553091292.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ase_sv_xx_3.1.0_2.4_1622553091292.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ase_sv| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Luba-Katanga BertForMaskedLM Cased model (from raduion) author: John Snow Labs name: bert_embeddings_medium_luxembourgish date: 2022-12-02 tags: [lu, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: lu edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-medium-luxembourgish` is a Luba-Katanga model originally trained by `raduion`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_medium_luxembourgish_lu_4.2.4_3.0_1670020644930.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_medium_luxembourgish_lu_4.2.4_3.0_1670020644930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_medium_luxembourgish","lu") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_medium_luxembourgish","lu") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_medium_luxembourgish| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|lu| |Size:|263.2 MB| |Case sensitive:|true| ## References - https://huggingface.co/raduion/bert-medium-luxembourgish --- layout: model title: English asr_Dansk_wav2vec2_stt TFWav2Vec2ForCTC from Siyam author: John Snow Labs name: pipeline_asr_Dansk_wav2vec2_stt date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Dansk_wav2vec2_stt` is a English model originally trained by Siyam. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_Dansk_wav2vec2_stt_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_Dansk_wav2vec2_stt_en_4.2.0_3.0_1664120521607.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_Dansk_wav2vec2_stt_en_4.2.0_3.0_1664120521607.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_Dansk_wav2vec2_stt', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_Dansk_wav2vec2_stt", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_Dansk_wav2vec2_stt| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese BertForQuestionAnswering Base Cased model (from jackh1995) author: John Snow Labs name: bert_qa_roberta_base_chinese_extractive date: 2022-07-07 tags: [zh, open_source, bert, question_answering] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-chinese-extractive-qa` is a Chinese model originally trained by `jackh1995`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_zh_4.0.0_3.0_1657190912340.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_roberta_base_chinese_extractive_zh_4.0.0_3.0_1657190912340.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_roberta_base_chinese_extractive","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_roberta_base_chinese_extractive","zh") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_roberta_base_chinese_extractive| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|zh| |Size:|381.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jackh1995/roberta-base-chinese-extractive-qa --- layout: model title: Legal Termination for cause Clause Binary Classifier (md) author: John Snow Labs name: legclf_termination_for_cause_md date: 2023-01-11 tags: [en, legal, classification, document, agreement, contract, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `termination-for-cause` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `termination-for-cause` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/CLASSIFY_LEGAL_DOCUMENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_termination_for_cause_md_en_1.0.0_3.0_1673460252721.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_termination_for_cause_md_en_1.0.0_3.0_1673460252721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_termination_for_cause_md", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[termination-for-cause]| |[other]| |[other]| |[termination-for-cause]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_termination_for_cause_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash precision recall f1-score support attorney-fees 1.00 1.00 1.00 38 other 1.00 1.00 1.00 39 accuracy 1.00 77 macro avg 1.00 1.00 1.00 77 weighted avg 1.00 1.00 1.00 77 ``` --- layout: model title: Detect Problems, Tests and Treatments (ner_clinical_en) author: John Snow Labs name: ner_clinical_en date: 2020-01-30 task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 tags: [clinical, licensed, ner, en] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terms. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities ``PROBLEM``, ``TEST``, ``TREATMENT``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL/){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_en_3.0.0_3.0_1617208419368.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_en_3.0.0_3.0_1617208419368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPython.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature."]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_clinical", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.").toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_21iridescent_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_21iridescent_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_21iridescent_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/21iridescent/distilbert-base-uncased-finetuned-squad --- layout: model title: T5 for Informal to Formal Style Transfer author: John Snow Labs name: t5_informal_to_formal_styletransfer date: 2022-01-12 tags: [t5, open_source, en] task: Text Generation language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a text-to-text model based on T5 fine-tuned to generate informal text from a formal text input, for the task "transfer Casual to Formal:". It is based on Prithiviraj Damodaran's Styleformer. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5_LINGUISTIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_informal_to_formal_styletransfer_en_3.4.0_3.0_1641985760876.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_informal_to_formal_styletransfer_en_3.4.0_3.0_1641985760876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from sparknlp.base import * from sparknlp.annotator import * spark = sparknlp.start() documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = T5Transformer.pretrained("t5_informal_to_formal_styletransfer") \ .setTask("transfer Casual to Formal:") \ .setInputCols(["documents"]) \ .setMaxOutputLength(200) \ .setOutputCol("transfers") pipeline = Pipeline().setStages([documentAssembler, t5]) data = spark.createDataFrame([["Who gives a crap?"]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("transfers.result").show(truncate=False) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.seq2seq.T5Transformer import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer.pretrained("t5_informal_to_formal_styletransfer") .setTask("transfer Casual to Formal:") .setMaxOutputLength(200) .setInputCols("documents") .setOutputCol("transfer") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("Who gives a crap?").toDF("text") val result = pipeline.fit(data).transform(data) result.select("transfer.result").show(false) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5.informal_to_formal_styletransfer").predict("""transfer Casual to Formal:""") ```

## Results ```bash +------------+ |result | +------------+ |[Who cares?]| +------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_informal_to_formal_styletransfer| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[summaries]| |Language:|en| |Size:|924.0 MB| ## Data Source The original model is from the transformers library: https://huggingface.co/prithivida/informal_to_formal_styletransfer --- layout: model title: Part of Speech for Polish author: John Snow Labs name: pos_ud_lfg date: 2021-03-08 tags: [part_of_speech, open_source, polish, pos_ud_lfg, pl] task: Part of Speech Tagging language: pl edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADJ - PROPN - VERB - NOUN - PUNCT - NUM - ADV - ADP - SCONJ - PRON - AUX - DET - CCONJ - PART - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_lfg_pl_3.0.0_3.0_1615230237237.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_lfg_pl_3.0.0_3.0_1615230237237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_lfg", "pl") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Witaj z John Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_lfg", "pl") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Witaj z John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Witaj z John Snow Labs! ""] token_df = nlu.load('pl.pos.ud_lfg').predict(text) token_df ```

## Results ```bash token pos 0 Witaj VERB 1 z ADP 2 John PROPN 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_lfg| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|pl| --- layout: model title: Legal Collateral Clause Binary Classifier author: John Snow Labs name: legclf_collateral_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `collateral` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `collateral` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_collateral_clause_en_1.0.0_3.2_1660123314449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_collateral_clause_en_1.0.0_3.2_1660123314449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_collateral_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[collateral]| |[other]| |[other]| |[collateral]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_collateral_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support collateral 0.92 0.89 0.90 37 other 0.94 0.96 0.95 69 accuracy - - 0.93 106 macro-avg 0.93 0.92 0.93 106 weighted-avg 0.93 0.93 0.93 106 ``` --- layout: model title: Extract relations between chemicals and proteins (ReDL) author: John Snow Labs name: redl_chemprot_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect interactions between chemicals and proteins using BERT model by classifying whether a specified semantic relation holds between the chemical and protein entities within a sentence or document. ## Predicted Entities `CPR:1`, `CPR:2`, `CPR:3`, `CPR:4`, `CPR:5`, `CPR:6`, `CPR:7`, `CPR:8`, `CPR:9`, `CPR:10` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CHEM_PROT){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_chemprot_biobert_en_2.7.3_2.4_1612443115083.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_chemprot_biobert_en_2.7.3_2.4_1612443115083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `redl_chemprot_biobert` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:---------------------:|:---------------------------------------------------------------------:|:---------------------:|---------------------------| | redl_chemprot_biobert | CPR:1, CPR:2, CPR:3, CPR:4, CPR:5, CPR:6, CPR:7, CPR:8, CPR:9, CPR:10 | ner_chemprot_clinical | [“No need to set pairs.”] |

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = sparknlp.annotators.Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") words_embedder = WordEmbeddingsModel() \ .pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"]) \ .setOutputCol("embeddings") ner_tagger = MedicalNerModel.pretrained("ner_chemprot_clinical", "en", "clinical/models")\ .setInputCols("sentences", "tokens", "embeddings")\ .setOutputCol("ner_tags") ) ner_converter = NerConverter() \ .setInputCols(["sentences", "tokens", "ner_tags"]) \ .setOutputCol("ner_chunks") dependency_parser = DependencyParserModel() \ .pretrained("dependency_conllu", "en") \ .setInputCols(["sentences", "pos_tags", "tokens"]) \ .setOutputCol("dependencies") # Set a filter on pairs of named entities which will be treated as relation candidates re_ner_chunk_filter = RENerChunksFilter() \ .setInputCols(["ner_chunks", "dependencies"])\ .setMaxSyntacticDistance(10)\ .setOutputCol("re_ner_chunks") #.setRelationPairs(['SYMPTOM-EXTERNAL_BODY_PART_OR_REGION']) # The dataset this model is trained to is sentence-wise. # This model can also be trained on document-level relations - in which case, while predicting, use "document" instead of "sentence" as input. re_model = RelationExtractionDLModel()\ .pretrained('redl_chemprot_biobert', 'en', "clinical/models") \ .setPredictionThreshold(0.5)\ .setInputCols(["re_ner_chunks", "sentences"]) \ .setOutputCol("relations") pipeline = Pipeline(stages=[documenter, sentencer, tokenizer, pos_tagger, words_embedder, ner_tagger, ner_converter, dependency_parser, re_ner_chunk_filter, re_model]) text='''In this study, we examined the effects of mitiglinide on various cloned K(ATP) channels (Kir6.2/SUR1, Kir6.2/SUR2A, and Kir6.2/SUR2B) reconstituted in COS-1 cells, and compared them to another meglitinide-related compound, nateglinide. Patch-clamp analysis using inside-out recording configuration showed that mitiglinide inhibits the Kir6.2/SUR1 channel currents in a dose-dependent manner (IC50 value, 100 nM) but does not significantly inhibit either Kir6.2/SUR2A or Kir6.2/SUR2B channel currents even at high doses (more than 10 microM). Nateglinide inhibits Kir6.2/SUR1 and Kir6.2/SUR2B channels at 100 nM, and inhibits Kir6.2/SUR2A channels at high concentrations (1 microM). Binding experiments on mitiglinide, nateglinide, and repaglinide to SUR1 expressed in COS-1 cells revealed that they inhibit the binding of [3H]glibenclamide to SUR1 (IC50 values: mitiglinide, 280 nM; nateglinide, 8 microM; repaglinide, 1.6 microM), suggesting that they all share a glibenclamide binding site. The insulin responses to glucose, mitiglinide, tolbutamide, and glibenclamide in MIN6 cells after chronic mitiglinide, nateglinide, or repaglinide treatment were comparable to those after chronic tolbutamide and glibenclamide treatment. These results indicate that, similar to the sulfonylureas, mitiglinide is highly specific to the Kir6.2/SUR1 complex, i.e., the pancreatic beta-cell K(ATP) channel, and suggest that mitiglinide may be a clinically useful anti-diabetic drug.''' p_model = pipeline.fit(spark.createDataFrame([[text]]).toDF("text")) result = p_model.transform(data) ``` ```scala ... val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentencer = SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val tokenizer = sparknlp.annotators.Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val pos_tagger = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val ner_tagger = MedicalNerModel.pretrained("ner_chemprot_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_converter = NerConverter() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") // Set a filter on pairs of named entities which will be treated as relation candidates val re_ner_chunk_filter = RENerChunksFilter() .setInputCols(Array("ner_chunks", "dependencies")) .setMaxSyntacticDistance(10) .setOutputCol("re_ner_chunks") // .setRelationPairs(Array("SYMPTOM-EXTERNAL_BODY_PART_OR_REGION")) // The dataset this model is trained to is sentence-wise. // This model can also be trained on document-level relations - in which case, while predicting, use "document" instead of "sentence" as input. val re_model = RelationExtractionDLModel() .pretrained("redl_chemprot_biobert", "en", "clinical/models") .setPredictionThreshold(0.5) .setInputCols(Array("re_ner_chunks", "sentences")) .setOutputCol("relations") val pipeline = new Pipeline().setStages(Array(documenter, sentencer, tokenizer, pos_tagger, words_embedder, ner_tagger, ner_converter, dependency_parser, re_ner_chunk_filter, re_model)) val data = Seq("In this study, we examined the effects of mitiglinide on various cloned K(ATP) channels (Kir6.2/SUR1, Kir6.2/SUR2A, and Kir6.2/SUR2B) reconstituted in COS-1 cells, and compared them to another meglitinide-related compound, nateglinide. Patch-clamp analysis using inside-out recording configuration showed that mitiglinide inhibits the Kir6.2/SUR1 channel currents in a dose-dependent manner (IC50 value, 100 nM) but does not significantly inhibit either Kir6.2/SUR2A or Kir6.2/SUR2B channel currents even at high doses (more than 10 microM). Nateglinide inhibits Kir6.2/SUR1 and Kir6.2/SUR2B channels at 100 nM, and inhibits Kir6.2/SUR2A channels at high concentrations (1 microM). Binding experiments on mitiglinide, nateglinide, and repaglinide to SUR1 expressed in COS-1 cells revealed that they inhibit the binding of [3H]glibenclamide to SUR1 (IC50 values: mitiglinide, 280 nM; nateglinide, 8 microM; repaglinide, 1.6 microM), suggesting that they all share a glibenclamide binding site. The insulin responses to glucose, mitiglinide, tolbutamide, and glibenclamide in MIN6 cells after chronic mitiglinide, nateglinide, or repaglinide treatment were comparable to those after chronic tolbutamide and glibenclamide treatment. These results indicate that, similar to the sulfonylureas, mitiglinide is highly specific to the Kir6.2/SUR1 complex, i.e., the pancreatic beta-cell K(ATP) channel, and suggest that mitiglinide may be a clinically useful anti-diabetic drug.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.chemprot").predict("""In this study, we examined the effects of mitiglinide on various cloned K(ATP) channels (Kir6.2/SUR1, Kir6.2/SUR2A, and Kir6.2/SUR2B) reconstituted in COS-1 cells, and compared them to another meglitinide-related compound, nateglinide. Patch-clamp analysis using inside-out recording configuration showed that mitiglinide inhibits the Kir6.2/SUR1 channel currents in a dose-dependent manner (IC50 value, 100 nM) but does not significantly inhibit either Kir6.2/SUR2A or Kir6.2/SUR2B channel currents even at high doses (more than 10 microM). Nateglinide inhibits Kir6.2/SUR1 and Kir6.2/SUR2B channels at 100 nM, and inhibits Kir6.2/SUR2A channels at high concentrations (1 microM). Binding experiments on mitiglinide, nateglinide, and repaglinide to SUR1 expressed in COS-1 cells revealed that they inhibit the binding of [3H]glibenclamide to SUR1 (IC50 values: mitiglinide, 280 nM; nateglinide, 8 microM; repaglinide, 1.6 microM), suggesting that they all share a glibenclamide binding site. The insulin responses to glucose, mitiglinide, tolbutamide, and glibenclamide in MIN6 cells after chronic mitiglinide, nateglinide, or repaglinide treatment were comparable to those after chronic tolbutamide and glibenclamide treatment. These results indicate that, similar to the sulfonylureas, mitiglinide is highly specific to the Kir6.2/SUR1 complex, i.e., the pancreatic beta-cell K(ATP) channel, and suggest that mitiglinide may be a clinically useful anti-diabetic drug.""") ```

## Results ```bash | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |---:|:-----------|:----------|----------------:|--------------:|:------------------|:----------|----------------:|--------------:|:--------------|-------------:| | 0 | CPR:2 | CHEMICAL | 43 | 53 | mitiglinide | GENE-N | 80 | 87 | channels | 0.998399 | | 1 | CPR:2 | GENE-N | 80 | 87 | channels | CHEMICAL | 224 | 234 | nateglinide | 0.994489 | | 2 | CPR:2 | CHEMICAL | 706 | 716 | mitiglinide | GENE-Y | 751 | 754 | SUR1 | 0.999304 | | 3 | CPR:2 | CHEMICAL | 823 | 839 | [3H]glibenclamide | GENE-Y | 844 | 847 | SUR1 | 0.998923 | | 4 | CPR:2 | GENE-N | 998 | 1004 | insulin | CHEMICAL | 1019 | 1025 | glucose | 0.979057 | | 5 | CPR:2 | GENE-N | 998 | 1004 | insulin | CHEMICAL | 1028 | 1038 | mitiglinide | 0.988504 | | 6 | CPR:2 | GENE-N | 998 | 1004 | insulin | CHEMICAL | 1041 | 1051 | tolbutamide | 0.991856 | | 7 | CPR:2 | GENE-N | 998 | 1004 | insulin | CHEMICAL | 1058 | 1070 | glibenclamide | 0.994092 | | 8 | CPR:2 | GENE-N | 998 | 1004 | insulin | CHEMICAL | 1100 | 1110 | mitiglinide | 0.994409 | | 9 | CPR:2 | CHEMICAL | 1290 | 1300 | mitiglinide | GENE-N | 1387 | 1393 | channel | 0.981534 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_chemprot_biobert| |Compatibility:|Healthcare NLP 2.7.3+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Data Source Trained on ChemProt benchmark dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support CPR:1 0.870 0.908 0.888 215 CPR:10 0.818 0.762 0.789 258 CPR:2 0.726 0.806 0.764 1651 CPR:3 0.788 0.785 0.787 657 CPR:4 0.901 0.855 0.878 1599 CPR:5 0.799 0.891 0.842 184 CPR:6 0.888 0.845 0.866 258 CPR:7 0.520 0.765 0.619 25 CPR:8 0.083 0.333 0.133 24 CPR:9 0.930 0.805 0.863 629 Avg. 0.732 0.775 0.743 ``` --- layout: model title: Catalan RobertaForQuestionAnswering Cased model (from crodri) author: John Snow Labs name: roberta_qa_ca_v2_squac_ca_catalan date: 2023-01-20 tags: [ca, open_source, roberta, question_answering, tensorflow] task: Question Answering language: ca edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-ca-v2-qa-squac-ca-catalanqa` is a Catalan model originally trained by `crodri`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ca_v2_squac_ca_catalan_ca_4.3.0_3.0_1674219909583.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ca_v2_squac_ca_catalan_ca_4.3.0_3.0_1674219909583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ca_v2_squac_ca_catalan","ca")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ca_v2_squac_ca_catalan","ca") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ca_v2_squac_ca_catalan| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|ca| |Size:|461.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/crodri/roberta-ca-v2-qa-squac-ca-catalanqa --- layout: model title: Translate Hiri Motu to English Pipeline author: John Snow Labs name: translate_ho_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, ho, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `ho` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_ho_en_xx_2.7.0_2.4_1609690127723.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_ho_en_xx_2.7.0_2.4_1609690127723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_ho_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_ho_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.ho.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_ho_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Russian BertForQuestionAnswering Cased model (from ruselkomp) author: John Snow Labs name: bert_qa_deep_pavlov_full_2 date: 2022-07-07 tags: [ru, open_source, bert, question_answering] task: Question Answering language: ru edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `deep-pavlov-full-2` is a Russian model originally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_deep_pavlov_full_2_ru_4.0.0_3.0_1657189320548.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_deep_pavlov_full_2_ru_4.0.0_3.0_1657189320548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_deep_pavlov_full_2","ru") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Как меня зовут?", "Меня зовут Клара, и я живу в Беркли."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_deep_pavlov_full_2","ru") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Как меня зовут?", "Меня зовут Клара, и я живу в Беркли.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_deep_pavlov_full_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ru| |Size:|665.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/deep-pavlov-full-2 --- layout: model title: BioBERT Sentence Embeddings (Clinical) author: John Snow Labs name: sent_biobert_clinical_base_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true recommended: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for generic clinical text. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_clinical_base_cased_en_2.6.2_2.4_1600533460155.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_clinical_base_cased_en_2.6.2_2.4_1600533460155.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_clinical_base_cased_embeddings I hate cancer [0.397987425327301, 0.6472950577735901, -0.551... Antibiotics aren't painkiller [0.19467104971408844, 0.3496762812137604, -0.3... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_clinical_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.2| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: English image_classifier_vit_garbage_classification ViTForImageClassification from yangy50 author: John Snow Labs name: image_classifier_vit_garbage_classification date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_garbage_classification` is a English model originally trained by yangy50. ## Predicted Entities `plastic`, `cardboard`, `paper`, `metal`, `glass`, `trash` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_garbage_classification_en_4.1.0_3.0_1660166313891.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_garbage_classification_en_4.1.0_3.0_1660166313891.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_garbage_classification", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_garbage_classification", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_garbage_classification| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Detect Company Sectors in texts (small) author: John Snow Labs name: finner_wiki_sector date: 2023-01-15 tags: [en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model, aimed to detect Company Sectors. It was trained with wikipedia texts about companies. ## Predicted Entities `SECTOR`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_wiki_sector_en_1.0.0_3.0_1673797361843.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_wiki_sector_en_1.0.0_3.0_1673797361843.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512) chunks = finance.NerConverterInternal()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") ner = finance.NerModel().pretrained("finner_wiki_sector", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") pipe = nlp.Pipeline(stages=[documenter, sentencizer, tokenizer, embeddings, ner, chunks]) model = pipe.fit(df) res = model.transform(df) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.ner_chunk.begin, res.ner_chunk.end, res.ner_chunk.metadata)).alias("cols")) \ .select(F.expr("cols['3']['sentence']").alias("sentence_id"), F.expr("cols['0']").alias("chunk"), F.expr("cols['2']").alias("end"), F.expr("cols['3']['entity']").alias("ner_label"))\ .filter("ner_label!='O'")\ .show(truncate=False) ```

## Results ```bash +-----------+-----------------+---+---------+ |sentence_id|chunk |end|ner_label| +-----------+-----------------+---+---------+ |1 |lawn mowers |175|SECTOR | |1 |snow blowers |192|SECTOR | |1 |irrigation system|214|SECTOR | +-----------+-----------------+---+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_wiki_sector| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.2 MB| ## References Wikipedia ## Benchmarking ```bash label tp fp fn prec rec f1 B-SECTOR 70 17 23 0.8045977 0.75268817 0.7777778 I-SECTOR 24 11 9 0.6857143 0.72727275 0.70588243 Macro-average 94 28 32 0.745156 0.73998046 0.7425592 Micro-average 94 32 0.7704918 0.74603176 0.7580645 ``` --- layout: model title: Mapping Companies to NASDAQ Stock Screener by Company Name author: John Snow Labs name: finmapper_nasdaq_company_name_stock_screener date: 2023-01-19 tags: [en, finance, licensed, nasdaq, company] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model allows you to, given an extracted name of a company, get following information about that company from Nasdaq Stock Screener: - Country - IPO_Year - Industry - Last_Sale - Market_Cap - Name - Net_Change - Percent_Change - Sector - Ticker - Volume It can be optionally combined with Entity Resolution to normalize first the name of the company. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_company_name_stock_screener_en_1.0.0_3.0_1674161310624.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_nasdaq_company_name_stock_screener_en_1.0.0_3.0_1674161310624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('document') tokenizer = nlp.Tokenizer()\ .setInputCols("document")\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained('finner_orgs_prods_alias', 'en', 'finance/models')\ .setInputCols(["document", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") # Optional: To normalize the ORG name using NASDAQ data before the mapping ########################################################################## chunkToDoc = nlp.Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") chunk_embeddings = nlp.UniversalSentenceEncoder.pretrained("tfhub_use", "en")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("chunk_embeddings") use_er_model = finance.SentenceEntityResolverModel.pretrained('finel_nasdaq_company_name_stock_screener', 'en', 'finance/models')\ .setInputCols("chunk_embeddings")\ .setOutputCol('normalized')\ .setDistanceFunction("EUCLIDEAN") ########################################################################## CM = finance.ChunkMapperModel.pretrained('finmapper_nasdaq_company_name_stock_screener', 'en', 'finance/models')\ .setInputCols(["normalized"])\ .setOutputCol("mappings") pipeline = nlp.Pipeline().setStages([document_assembler, tokenizer, embeddings, ner_model, ner_converter, chunkToDoc, # Optional for normalization chunk_embeddings, # Optional for normalization use_er_model, # Optional for normalization CM]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = pipeline.fit(empty_data) lp = nlp.LightPipeline(model) text = """Nike is an American multinational association that is involved in the design, development, manufacturing and worldwide marketing and sales of apparel, footwear, accessories, equipment and services.""" result = lp.fullAnnotate(text) ```

## Results ```bash "Country": "United States", "IPO_Year": "0", "Industry": "Shoe Manufacturing", "Last_Sale": "$128.85", "Market_Cap": "1.9979004036E11", "Name": "Nike Inc. Common Stock", "Net_Change": "0.96", "Percent_Change": "0.751%", "Sector": "Consumer Discretionary", "Symbol": "NKE", "Volume": "4854668" ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_nasdaq_company_name_stock_screener| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|599.1 KB| ## References https://www.nasdaq.com/market-activity/stocks/screener --- layout: model title: Translate English to Tonga (Zambezi) Pipeline author: John Snow Labs name: translate_en_toi date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, toi, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `toi` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_toi_xx_2.7.0_2.4_1609689111234.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_toi_xx_2.7.0_2.4_1609689111234.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_toi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_toi", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.toi').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_toi| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Seychellois Creole to English author: John Snow Labs name: opus_mt_crs_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, crs, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `crs` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_crs_en_xx_2.7.0_2.4_1609168963014.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_crs_en_xx_2.7.0_2.4_1609168963014.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_crs_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Spanish RobertaForTokenClassification Base Cased model (from bertin-project) author: John Snow Labs name: roberta_token_classifier_bertin_base_pos_conll2002 date: 2023-03-01 tags: [es, open_source, roberta, token_classification, ner, tensorflow] task: Named Entity Recognition language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertin-base-pos-conll2002-es` is a Spanish model originally trained by `bertin-project`. ## Predicted Entities `DA`, `VAM`, `I`, `VSM`, `PP`, `VSS`, `DI`, `AQ`, `Y`, `VMN`, `Fit`, `Fg`, `Fia`, `Fpa`, `Fat`, `VSN`, `Fpt`, `DD`, `VAP`, `SP`, `NP`, `Fh`, `VAI`, `CC`, `Fd`, `VMG`, `NC`, `PX`, `DE`, `Fz`, `PN`, `Fx`, `Faa`, `Fs`, `Fe`, `VSP`, `DP`, `VAS`, `VSG`, `PT`, `Ft`, `VAN`, `PI`, `P0`, `RG`, `RN`, `CS`, `DN`, `VMI`, `Fp`, `Fc`, `PR`, `VSI`, `AO`, `VMM`, `PD`, `VMS`, `DT`, `Z`, `VMP` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bertin_base_pos_conll2002_es_4.3.0_3.0_1677703697571.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_bertin_base_pos_conll2002_es_4.3.0_3.0_1677703697571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_bertin_base_pos_conll2002","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = RobertaForTokenClassification.pretrained("roberta_token_classifier_bertin_base_pos_conll2002","es") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_bertin_base_pos_conll2002| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|426.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/bertin-project/bertin-base-pos-conll2002-es --- layout: model title: Fast Neural Machine Translation Model from English to Spanish author: John Snow Labs name: opus_mt_en_es date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, es, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `es` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_es_xx_2.7.0_2.4_1609167574367.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_es_xx_2.7.0_2.4_1609167574367.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_es| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: French CamemBert Embeddings (from codingJacob) author: John Snow Labs name: camembert_embeddings_codingJacob_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `codingJacob`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_codingJacob_generic_model_fr_3.4.4_3.0_1653987610803.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_codingJacob_generic_model_fr_3.4.4_3.0_1653987610803.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_codingJacob_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/codingJacob/dummy-model --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-few-shot-k-64-finetuned-squad-seed-0` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0_en_4.0.0_3.0_1654180885453.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0_en_4.0.0_3.0_1654180885453.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_64d_seed_0").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_few_shot_k_64_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-base-uncased-few-shot-k-64-finetuned-squad-seed-0 --- layout: model title: English image_classifier_vit_new_exper3 ViTForImageClassification from sudo-s author: John Snow Labs name: image_classifier_vit_new_exper3 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_new_exper3` is a English model originally trained by sudo-s. ## Predicted Entities `45`, `98`, `113`, `34`, `67`, `120`, `93`, `142`, `147`, `12`, `66`, `89`, `51`, `124`, `84`, `8`, `73`, `78`, `19`, `100`, `23`, `62`, `135`, `128`, `4`, `121`, `88`, `77`, `40`, `110`, `15`, `11`, `104`, `90`, `9`, `141`, `139`, `132`, `44`, `33`, `117`, `22`, `56`, `55`, `26`, `134`, `50`, `123`, `37`, `68`, `61`, `107`, `13`, `46`, `99`, `24`, `94`, `83`, `35`, `16`, `79`, `5`, `103`, `112`, `72`, `10`, `59`, `144`, `87`, `48`, `21`, `116`, `76`, `138`, `54`, `43`, `148`, `127`, `65`, `71`, `57`, `108`, `32`, `80`, `106`, `137`, `82`, `49`, `6`, `126`, `36`, `1`, `39`, `140`, `17`, `25`, `60`, `14`, `133`, `47`, `122`, `111`, `102`, `31`, `96`, `69`, `95`, `58`, `145`, `64`, `53`, `42`, `75`, `115`, `109`, `0`, `20`, `27`, `70`, `2`, `86`, `38`, `81`, `118`, `92`, `125`, `18`, `101`, `30`, `7`, `143`, `97`, `130`, `114`, `129`, `29`, `41`, `105`, `63`, `3`, `74`, `91`, `52`, `85`, `131`, `28`, `119`, `136`, `146` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_new_exper3_en_4.1.0_3.0_1660173387567.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_new_exper3_en_4.1.0_3.0_1660173387567.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_new_exper3", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_new_exper3", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_new_exper3| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.3 MB| --- layout: model title: Detect Time-related Terminology author: John Snow Labs name: roberta_token_classifier_timex_semeval date: 2021-12-28 tags: [timex, ner, roberta, en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.3.4 spark_version: 2.4 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model was imported from `Hugging Face` and it's been trained to detect time-related terminology, leveraging `RoBERTa` embeddings and `RobertaForTokenClassification` for NER purposes. ## Predicted Entities `Period`, `Year`, `Calendar-Interval`, `Month-Of-Year`, `Day-Of-Month`, `Day-Of-Week`, `Hour-Of-Day`, `Minute-Of-Hour`, `Number`, `Second-Of-Minute`, `Time-Zone`, `Part-Of-Day`, `Season-Of-Year`, `AMPM-Of-Day`, `Part-Of-Week`, `Week-Of-Year`, `Two-Digit-Year`, `Sum`, `Difference`, `Union`, `Intersection`, `Every-Nth`, `This`, `Last`, `Next`, `Before`, `After`, `Between`, `NthFromStart`, `NthFromEnd`, `Frequency`, `Modifier` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_TIMEX_SEMEVAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_timex_semeval_en_3.3.4_2.4_1640679857852.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_token_classifier_timex_semeval_en_3.3.4_2.4_1640679857852.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+-----------------+ |chunk |ner_label | +-------+-----------------+ |22:12C |Period | |3 |Number | |days |Calendar-Interval| |Tuesday|Day-Of-Week | |to |Between | |Friday |Day-Of-Week | +-------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_token_classifier_timex_semeval| |Compatibility:|Spark NLP 3.3.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|en| |Size:|439.5 MB| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/clulab/roberta-timex-semeval](https://huggingface.co/clulab/roberta-timex-semeval) --- layout: model title: Relation Extraction between different oncological entity types using granular classes (ReDL) author: John Snow Labs name: redl_oncology_granular_biobert_wip date: 2022-09-29 tags: [licensed, clinical, oncology, en, relation_extraction, temporal, test, biomarker, anatomy] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Using this relation extraction model, four relation types can be identified: is_date_of (between date entities and other clinical entities), is_size_of (between Tumor_Finding and Tumor_Size), is_location_of (between anatomical entities and other entities) and is_finding_of (between test entities and their results). ## Predicted Entities `is_date_of`, `is_finding_of`, `is_location_of`, `is_size_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_granular_biobert_wip_en_4.1.0_3.0_1664482477934.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_granular_biobert_wip_en_4.1.0_3.0_1664482477934.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash | chunk1 | entity1 | chunk2 | entity2 | relation | confidence | |:-----------|:---------------|:---------------|:--------------|:-----------|-------------:| | mastectomy | Cancer_Surgery | two months ago | Relative_Date | is_date_of | 0.965252 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_granular_biobert_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.83 0.91 0.87 is_date_of 0.82 0.80 0.81 is_finding_of 0.92 0.85 0.88 is_location_of 0.95 0.85 0.90 is_size_of 0.91 0.80 0.85 macro-avg 0.89 0.84 0.86 ``` --- layout: model title: Legal Cultivation Of Agricultural Land Document Classifier (EURLEX) author: John Snow Labs name: legclf_cultivation_of_agricultural_land_bert date: 2023-03-06 tags: [en, legal, classification, clauses, cultivation_of_agricultural_land, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_cultivation_of_agricultural_land_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Cultivation_of_Agricultural_Land or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Cultivation_of_Agricultural_Land`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cultivation_of_agricultural_land_bert_en_1.0.0_3.0_1678111585218.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cultivation_of_agricultural_land_bert_en_1.0.0_3.0_1678111585218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_cultivation_of_agricultural_land_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Cultivation_of_Agricultural_Land]| |[Other]| |[Other]| |[Cultivation_of_Agricultural_Land]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cultivation_of_agricultural_land_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.1 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Cultivation_of_Agricultural_Land 0.86 0.88 0.87 56 Other 0.87 0.85 0.86 55 accuracy - - 0.86 111 macro-avg 0.87 0.86 0.86 111 weighted-avg 0.86 0.86 0.86 111 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Tuvaluan author: John Snow Labs name: opus_mt_en_tvl date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tvl, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tvl` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tvl_xx_2.7.0_2.4_1609168063147.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tvl_xx_2.7.0_2.4_1609168063147.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tvl| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English XLMRobertaForTokenClassification Base Cased model (from Yaxin) author: John Snow Labs name: xlmroberta_ner_yaxin_base_conll2003 date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-conll2003-ner` is a English model originally trained by `Yaxin`. ## Predicted Entities `ORG`, `LOC`, `PER`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_yaxin_base_conll2003_en_4.1.0_3.0_1660426515795.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_yaxin_base_conll2003_en_4.1.0_3.0_1660426515795.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_yaxin_base_conll2003","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_yaxin_base_conll2003","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_yaxin_base_conll2003| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|792.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Yaxin/xlm-roberta-base-conll2003-ner - https://paperswithcode.com/sota?task=Token+Classification&dataset=conll2003 --- layout: model title: Pipeline to Resolve Medication Codes author: John Snow Labs name: medication_resolver_pipeline date: 2023-04-13 tags: [licensed, clinical, en, resolver, snomed, umls, rxnorm, ndc, ade, pipeline] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pretrained resolver pipeline to extract medications and resolve their adverse reactions (ADE), RxNorm, UMLS, NDC, SNOMED CT codes, and action/treatments in clinical text. Action/treatments are available for branded medication, and SNOMED codes are available for non-branded medication. This pipeline can be used as Lightpipeline (with `annotate/fullAnnotate`). You can use `medication_resolver_transform_pipeline` for Spark transform. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/medication_resolver_pipeline_en_4.3.2_3.0_1681388823359.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/medication_resolver_pipeline_en_4.3.2_3.0_1681388823359.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | entities | ADE | RxNorm | Action | Treatment | UMLS | SNOMED_CT | NDC_Product | NDC_Package | |---:|:-----------------------------|:-----------|:----------------------------|---------:|:---------------------------|:-------------------------------------------|:---------|:------------|:--------------|:--------------| | 0 | Amlodopine Vallarta 10-320mg | DRUG | Gynaecomastia | 722131 | NONE | NONE | C1949334 | 425838008 | 00093-7693 | 00093-7693-56 | | 1 | Eviplera | DRUG | Anxiety | 217010 | Inhibitory Bone Resorption | Osteoporosis | C0720318 | NONE | NONE | NONE | | 2 | Lescol 40 MG | DRUG | NONE | 103919 | Hypocholesterolemic | Heterozygous Familial Hypercholesterolemia | C0353573 | NONE | 00078-0234 | 00078-0234-05 | | 3 | Everolimus 1.5 mg tablet | DRUG | Acute myocardial infarction | 2056895 | NONE | NONE | C4723581 | NONE | 00054-0604 | 00054-0604-21 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|medication_resolver_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.1 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel - TextMatcherModel - ChunkMergeModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperFilterer - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel - ResolverMerger - ResolverMerger - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - ChunkMapperModel - Finisher --- layout: model title: Bangla Bert Embeddings author: John Snow Labs name: bert_embeddings_bangla_bert_base date: 2022-04-11 tags: [bert, embeddings, bn, open_source] task: Embeddings language: bn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bangla-bert-base` is a Bangla model orginally trained by `sagorsarker`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_bert_base_bn_3.4.2_3.0_1649673290861.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bangla_bert_base_bn_3.4.2_3.0_1649673290861.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bangla_bert_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|bn| |Size:|617.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/sagorsarker/bangla-bert-base - https://github.com/sagorbrur/bangla-bert - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://oscar-corpus.com/ - https://dumps.wikimedia.org/bnwiki/latest/ - https://github.com/sagorbrur/bnlp - https://github.com/sagorbrur/bangla-bert - https://github.com/google-research/bert - https://twitter.com/mapmeld - https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP - https://github.com/sagorbrur/bangla-bert/blob/master/notebook/bangla-bert-evaluation-classification-task.ipynb - https://github.com/sagorbrur/bangla-bert/tree/master/evaluations/wikiann - https://arxiv.org/abs/2012.14353 - https://arxiv.org/abs/2104.08613 - https://arxiv.org/abs/2107.03844 - https://arxiv.org/abs/2101.00204 - https://github.com/sagorbrur - https://www.tensorflow.org/tfrc - https://github.com/google-research/bert --- layout: model title: English BertForQuestionAnswering model (from nbroad) author: John Snow Labs name: bert_qa_xdistil_l12_h384_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xdistil-l12-h384-squad2` is a English model orginally trained by `nbroad`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_xdistil_l12_h384_squad2_en_4.0.0_3.0_1654192565150.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_xdistil_l12_h384_squad2_en_4.0.0_3.0_1654192565150.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_xdistil_l12_h384_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_xdistil_l12_h384_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.distilled").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_xdistil_l12_h384_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|124.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nbroad/xdistil-l12-h384-squad2 --- layout: model title: Legal Reimbursements Clause Binary Classifier author: John Snow Labs name: legclf_reimbursements_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `reimbursements` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `reimbursements` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_reimbursements_clause_en_1.0.0_3.2_1660123924164.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_reimbursements_clause_en_1.0.0_3.2_1660123924164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_reimbursements_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[reimbursements]| |[other]| |[other]| |[reimbursements]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_reimbursements_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.97 0.96 132 reimbursements 0.93 0.90 0.91 58 accuracy - - 0.95 190 macro-avg 0.94 0.93 0.94 190 weighted-avg 0.95 0.95 0.95 190 ``` --- layout: model title: Pipeline to Detect Chemicals and Proteins in text (biobert) author: John Snow Labs name: ner_chemprot_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chemprot_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_chemprot_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_biobert_pipeline_en_4.3.0_3.2_1679314581092.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_biobert_pipeline_en_4.3.0_3.2_1679314581092.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_chemprot_biobert_pipeline", "en", "clinical/models") text = '''Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_chemprot_biobert_pipeline", "en", "clinical/models") val text = "Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemprot_biobert.pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:------------|-------------:| | 0 | Keratinocyte | 0 | 11 | GENE-Y | 0.894 | | 1 | growth | 13 | 18 | GENE-Y | 0.4833 | | 2 | factor | 20 | 25 | GENE-Y | 0.7991 | | 3 | acidic | 31 | 36 | GENE-Y | 0.9765 | | 4 | fibroblast | 38 | 47 | GENE-Y | 0.3905 | | 5 | growth | 49 | 54 | GENE-Y | 0.7109 | | 6 | factor | 56 | 61 | GENE-Y | 0.8693 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemprot_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.2 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: English asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you TFWav2Vec2ForCTC from project2you author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you` is a English model originally trained by project2you. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_en_4.2.0_3.0_1664110092661.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you_en_4.2.0_3.0_1664110092661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_demo_colab_by_project2you| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Tagalog Electra Embeddings (from jcblaise) author: John Snow Labs name: electra_embeddings_electra_tagalog_base_cased_generator date: 2022-05-17 tags: [tl, open_source, electra, embeddings] task: Embeddings language: tl edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-tagalog-base-cased-generator` is a Tagalog model orginally trained by `jcblaise`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_base_cased_generator_tl_3.4.4_3.0_1652786728634.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_tagalog_base_cased_generator_tl_3.4.4_3.0_1652786728634.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_tagalog_base_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tl| |Size:|130.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/jcblaise/electra-tagalog-base-cased-generator - https://blaisecruz.com --- layout: model title: Detect PHI for Deidentification (Enriched) author: John Snow Labs name: ner_deid_enriched date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Named Entity recognition annotator allows for a generic model to be trained by utilizing a deep learning algorithm (Char CNNs - BiLSTM - CRF - word embeddings) inspired on a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM,CNN.Deidentification NER (Enriched) is a Named Entity Recognition model that annotates text to find protected health information that may need to be deidentified. The entities it annotates are Age, City, Country, Date, Doctor, Hospital, Idnum, Medicalrecord, Organization, Patient, Phone, Profession, State, Street, Username, and Zip. Clinical NER is trained with the 'embeddings_clinical' word embeddings model, so be sure to use the same embeddings in the pipeline. We sticked to official annotation guideline (AG) for 2014 i2b2 Deid challenge while annotating new datasets for this model. All the details regarding the nuances and explanations for AG can be found here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978170/) ## Predicted Entities ``Age``, ``City``, ``Country``, ``Date``, ``Doctor``, ``Hospital``, ``Idnum``, ``Medicalrecord``, ``Organization``, ``Patient``, ``Phone``, ``Profession``, ``State``, ``Street``, ``Username``, ``Zip`` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_DEMOGRAPHICS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_DEMOGRAPHICS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_en_3.0.0_3.0_1617208426129.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_deid_enriched_en_3.0.0_3.0_1617208426129.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained("ner_deid_enriched","en","clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_deid_enriched","en","clinical/models") .setInputCols("sentence","token","embeddings") .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.deid.enriched").predict("""HISTORY OF PRESENT ILLNESS: Mr. Smith is a 60-year-old white male veteran with multiple comorbidities, who has a history of bladder cancer diagnosed approximately two years ago by the VA Hospital. He underwent a resection there. He was to be admitted to the Day Hospital for cystectomy. He was seen in Urology Clinic and Radiology Clinic on 02/04/2003. HOSPITAL COURSE: Mr. Smith presented to the Day Hospital in anticipation for Urology surgery. On evaluation, EKG, echocardiogram was abnormal, a Cardiology consult was obtained. A cardiac adenosine stress MRI was then proceeded, same was positive for inducible ischemia, mild-to-moderate inferolateral subendocardial infarction with peri-infarct ischemia. In addition, inducible ischemia seen in the inferior lateral septum. Mr. Smith underwent a left heart catheterization, which revealed two vessel coronary artery disease. The RCA, proximal was 95% stenosed and the distal 80% stenosed. The mid LAD was 85% stenosed and the distal LAD was 85% stenosed. There was four Multi-Link Vision bare metal stents placed to decrease all four lesions to 0%. Following intervention, Mr. Smith was admitted to 7 Ardmore Tower under Cardiology Service under the direction of Dr. Hart. Mr. Smith had a noncomplicated post-intervention hospital course. He was stable for discharge home on 02/07/2003 with instructions to take Plavix daily for one month and Urology is aware of the same.""") ```

## Results ```bash +---------------+---------+ |chunk |ner_label| +---------------+---------+ |Smith |PATIENT | |VA Hospital |HOSPITAL | |Day Hospital |HOSPITAL | |02/04/2003 |DATE | |Smith |PATIENT | |Day Hospital |HOSPITAL | |Smith |PATIENT | |Smith |PATIENT | |7 Ardmore Tower|HOSPITAL | |Hart |DOCTOR | |Smith |PATIENT | |02/07/2003 |DATE | +---------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_deid_enriched| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on JSL enriched n2c2 2014: De-identification and Heart Disease Risk Factors Challenge datasets with `embeddings_clinical` https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/ ## Benchmarking ```bash label tp fp fn prec rec f1 I-AGE 7 3 6 0.7 0.538462 0.608696 I-DOCTOR 800 27 94 0.967352 0.894855 0.929692 I-IDNUM 6 0 2 1 0.75 0.857143 B-DATE 1883 34 56 0.982264 0.971119 0.97666 I-DATE 425 28 25 0.93819 0.944444 0.941307 B-PHONE 29 7 9 0.805556 0.763158 0.783784 B-STATE 87 4 11 0.956044 0.887755 0.920635 B-CITY 35 11 26 0.76087 0.57377 0.654206 I-ORGANIZATION 12 4 15 0.75 0.444444 0.55814 B-DOCTOR 728 75 53 0.9066 0.932138 0.919192 I-PROFESSION 43 11 13 0.796296 0.767857 0.781818 I-PHONE 62 4 4 0.939394 0.939394 0.939394 B-AGE 234 13 16 0.947368 0.936 0.94165 B-STREET 20 7 16 0.740741 0.555556 0.634921 I-ZIP 60 3 2 0.952381 0.967742 0.96 I-MEDICALRECORD 54 5 2 0.915254 0.964286 0.93913 B-ZIP 2 1 0 0.666667 1 0.8 B-HOSPITAL 256 23 66 0.917563 0.795031 0.851913 I-STREET 150 17 20 0.898204 0.882353 0.890208 B-COUNTRY 22 2 8 0.916667 0.733333 0.814815 I-COUNTRY 1 0 0 1 1 1 I-STATE 6 0 1 1 0.857143 0.923077 B-USERNAME 30 0 4 1 0.882353 0.9375 I-HOSPITAL 295 37 64 0.888554 0.821727 0.853835 I-PATIENT 243 26 41 0.903346 0.855634 0.878843 B-PROFESSION 52 8 17 0.866667 0.753623 0.806202 B-IDNUM 32 3 12 0.914286 0.727273 0.810127 I-CITY 76 15 13 0.835165 0.853933 0.844444 B-PATIENT 337 29 40 0.920765 0.893899 0.907133 B-MEDICALRECORD 74 6 4 0.925 0.948718 0.936709 B-ORGANIZATION 20 5 13 0.8 0.606061 0.689655 Macro-average 6083 408 673 0.7976 0.697533 0.744218 Micro-average 6083 408 673 0.937144 0.900385 0.918397 ``` --- layout: model title: English Bert Embeddings (Base, Uncased, Chemical) author: John Snow Labs name: bert_embeddings_chemical_bert_uncased date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chemical-bert-uncased` is a English model orginally trained by `recobo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chemical_bert_uncased_en_3.4.2_3.0_1649671988862.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chemical_bert_uncased_en_3.4.2_3.0_1649671988862.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chemical_bert_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|412.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/recobo/chemical-bert-uncased --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from cj-mills) author: John Snow Labs name: xlmroberta_ner_cj_mills_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `cj-mills`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cj_mills_base_finetuned_panx_de_4.1.0_3.0_1660431546170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_cj_mills_base_finetuned_panx_de_4.1.0_3.0_1660431546170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cj_mills_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_cj_mills_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_cj_mills_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|850.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/cj-mills/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-27 tags: [ro, licensed, ner, legal, mapa] task: Named Entity Recognition language: ro edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Romanian` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_ro_1.0.0_3.0_1682609352989.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_ro_1.0.0_3.0_1682609352989.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_base_ro_cased", "ro")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "ro", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Or, rezultă din hotărârea Curții de Apel București din 12 iunie 2013 că instanța română a aplicat greșit dreptul Uniunii (32) atunci când a respins excepția de litispendență invocată de domnul Liberato, întemeiată pe cererile referitoare la legătura matrimonială."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +---------------+---------+ |chunk |ner_label| +---------------+---------+ |București |ADDRESS | |12 iunie 2013 |DATE | |domnul Liberato|PERSON | +---------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.88 0.96 0.92 23 AMOUNT 1.00 0.67 0.80 3 DATE 0.97 0.97 0.97 31 ORGANISATION 0.67 0.71 0.69 28 PERSON 0.91 0.83 0.87 48 macro-avg 0.86 0.86 0.86 133 macro-avg 0.88 0.83 0.85 133 weighted-avg 0.87 0.86 0.86 133 ``` --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4_en_4.0.0_3.0_1655731258207.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4_en_4.0.0_3.0_1655731258207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_128d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|423.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-4 --- layout: model title: Legal No Conflicts Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_no_conflicts_bert date: 2023-03-05 tags: [en, legal, classification, clauses, no_conflicts, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `No_Conflicts` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `No_Conflicts`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_conflicts_bert_en_1.0.0_3.0_1678050654719.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_conflicts_bert_en_1.0.0_3.0_1678050654719.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_no_conflicts_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[No_Conflicts]| |[Other]| |[Other]| |[No_Conflicts]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_conflicts_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support No_Conflicts 0.96 0.98 0.97 84 Other 0.98 0.97 0.98 118 accuracy - - 0.98 202 macro-avg 0.97 0.98 0.97 202 weighted-avg 0.98 0.98 0.98 202 ``` --- layout: model title: Detect Adverse Drug Events (clinical_medium) author: John Snow Labs name: ner_ade_emb_clinical_medium date: 2023-05-21 tags: [en, clinical, ade, drug, licensed, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse reactions to drugs in reviews, tweets, and medical text using a pre-trained NER model. ## Predicted Entities `DRUG`, `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_ade_emb_clinical_medium_en_4.4.2_3.0_1684646733993.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_ade_emb_clinical_medium_en_4.4.2_3.0_1684646733993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_ade_emb_clinical_medium", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverterInternal()\ .setInputCols(['sentence', 'token', 'ner'])\ .setOutputCol('ner_chunk') pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter ]) sample_df = spark.createDataFrame([["Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps."]]).toDF("text") result = pipeline.fit(sample_df).transform(sample_df) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val clinical_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical_medium", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_ade_emb_clinical_medium", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverterInternal() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, clinical_embeddings, ner_model, ner_converter)) val sample_data = Seq("Been taking Lipitor for 15 years , have experienced severe fatigue a lot!!! . Doctor moved me to voltaren 2 months ago , so far , have only experienced cramps.").toDS.toDF("text") val result = pipeline.fit(sample_data).transform(sample_data) ```

## Results ```bash +--------------+-----+---+---------+ |chunk |begin|end|ner_label| +--------------+-----+---+---------+ |Lipitor |12 |18 |DRUG | |severe fatigue|52 |65 |ADE | |voltaren |97 |104|DRUG | |cramps |152 |157|ADE | +--------------+-----+---+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ade_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.7 MB| ## Benchmarking ```bash label precision recall f1-score support DRUG 0.92 0.91 0.91 15895 ADE 0.83 0.77 0.80 6077 micro-avg 0.89 0.87 0.88 21972 macro-avg 0.87 0.84 0.86 21972 weighted-avg 0.89 0.87 0.88 21972 ``` --- layout: model title: Extract treatment entities (Voice of the Patients) author: John Snow Labs name: ner_vop_treatment_wip date: 2023-05-19 tags: [licensed, clinical, en, ner, vop, patient, treatment] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts treatments mentioned in documents transferred from the patient’s own sentences. Note: ‘wip’ suffix indicates that the model development is work-in-progress and will be finalised and the model performance will improved in the upcoming releases. ## Predicted Entities `Drug`, `Form`, `Route`, `Dosage`, `Duration`, `Procedure`, `Frequency`, `Treatment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_wip_en_4.4.2_3.0_1684513558207.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_treatment_wip_en_4.4.2_3.0_1684513558207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:----------|:------------| | metformin | Drug | | glipizide | Drug | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_treatment_wip| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.9 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Drug 1299 92 141 1440 0.93 0.90 0.92 Form 251 36 15 266 0.87 0.94 0.91 Route 41 5 7 48 0.89 0.85 0.87 Dosage 342 35 70 412 0.91 0.83 0.87 Duration 2064 530 246 2310 0.80 0.89 0.84 Procedure 543 67 162 705 0.89 0.77 0.83 Frequency 888 181 191 1079 0.83 0.82 0.83 Treatment 165 54 63 228 0.75 0.72 0.74 macro_avg 5593 1000 895 6488 0.86 0.84 0.85 micro_avg 5593 1000 895 6488 0.85 0.86 0.86 ``` --- layout: model title: Legal Participation Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_participation_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, participation, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_participation_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `participation-agreement` for similar document type classification) or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `participation-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_participation_agreement_bert_en_1.0.0_3.0_1669368597593.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_participation_agreement_bert_en_1.0.0_3.0_1669368597593.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_participation_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[participation-agreement]| |[other]| |[other]| |[participation-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_participation_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.97 1.00 0.98 65 participation-agreement 1.00 0.94 0.97 33 accuracy - - 0.98 98 macro-avg 0.99 0.97 0.98 98 weighted-avg 0.98 0.98 0.98 98 ``` --- layout: model title: Arabic Named Entity Recognition (Modern Standard Arabic-MSA) author: John Snow Labs name: bert_ner_bert_base_arabic_camelbert_msa_ner date: 2022-05-04 tags: [bert, ner, token_classification, ar, open_source] task: Named Entity Recognition language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-msa-ner` is a Arabic model orginally trained by `CAMeL-Lab`. ## Predicted Entities `ORG`, `LOC`, `PERS`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_arabic_camelbert_msa_ner_ar_3.4.2_3.0_1651630233283.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_arabic_camelbert_msa_ner_ar_3.4.2_3.0_1651630233283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_arabic_camelbert_msa_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-msa-ner - https://camel.abudhabi.nyu.edu/anercorp/ - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: BERT Sequence Classification - Detect Spam SMS author: John Snow Labs name: bert_sequence_classifier_sms_spam date: 2021-11-07 tags: [sms, spam, bert_for_sequence_classification, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true annotator: BertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is imported from `Hugging Face-models`. It is a BERT-Tiny version of the `sms_spam` dataset. It identifies if the SMS is spam or not. - `LABEL_0` : No Spam - `LABEL_1` : Spam ## Predicted Entities `LABEL_0`, `LABEL_1` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_sms_spam_en_3.3.2_2.4_1636290194115.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_sequence_classifier_sms_spam_en_3.3.2_2.4_1636290194115.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['LABEL_1'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_sms_spam| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[label]| |Language:|en| |Case sensitive:|true| ## Data Source [https://huggingface.co/mrm8488/bert-tiny-finetuned-sms-spam-detection](https://huggingface.co/mrm8488/bert-tiny-finetuned-sms-spam-detection) ## Benchmarking ```bash label score accuracy 0.98 ``` --- layout: model title: Legal Wood Industry Document Classifier (EURLEX) author: John Snow Labs name: legclf_wood_industry_bert date: 2023-03-06 tags: [en, legal, classification, clauses, wood_industry, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_wood_industry_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Wood_Industry or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Wood_Industry`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_wood_industry_bert_en_1.0.0_3.0_1678111777594.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_wood_industry_bert_en_1.0.0_3.0_1678111777594.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_wood_industry_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Wood_Industry]| |[Other]| |[Other]| |[Wood_Industry]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_wood_industry_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.94 0.88 0.91 34 Wood_Industry 0.86 0.92 0.89 26 accuracy - - 0.90 60 macro-avg 0.90 0.90 0.90 60 weighted-avg 0.90 0.90 0.90 60 ``` --- layout: model title: Pipeline to Mapping ICD10-CM Codes with Their Corresponding SNOMED Codes author: John Snow Labs name: icd10cm_snomed_mapping date: 2023-03-29 tags: [en, licensed, icd10cm, snomed, pipeline, chunk_mapping] task: Chunk Mapping language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `icd10cm_snomed_mapper` model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_snomed_mapping_en_4.3.2_3.2_1680118771671.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_snomed_mapping_en_4.3.2_3.2_1680118771671.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd10cm_snomed_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(R079 N4289 M62830) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10cm_snomed_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(R079 N4289 M62830) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.icd10cm_to_snomed.pipe").predict("""Put your text here.""") ```

## Results ```bash | | icd10cm_code | snomed_code | |---:|:----------------------|:-----------------------------------------| | 0 | R079 | N4289 | M62830 | 161972006 | 22035000 | 16410651000119105 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_snomed_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.1 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Entity Resolver for Human Phenotype Ontology author: John Snow Labs name: sbiobertresolve_HPO date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps phenotypic abnormalities encountered in human diseases to Human Phenotype Ontology (HPO) codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities This model returns Human Phenotype Ontology (HPO) codes for phenotypic abnormalities encountered in human diseases. It also returns associated codes from the following vocabularies for each HPO code: - MeSH (Medical Subject Headings)- SNOMED- UMLS (Unified Medical Language System ) - ORPHA (international reference resource for information on rare diseases and orphan drugs) - OMIM (Online Mendelian Inheritance in Man) {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_MSH/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_HPO_en_3.0.4_3.0_1621189482944.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_HPO_en_3.0.4_3.0_1621189482944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_HPO``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_human_phenotype_gene_clinical``` as NER model. No need to ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \\ .setInputCol("text") \\ .setOutputCol("documents") gpt2 = GPT2Transformer.pretrained("gpt2_distilled") \\ .setInputCols(["documents"]) \\ .setMaxOutputLength(50) \\ .setOutputCol("generation") pipeline = Pipeline().setStages([documentAssembler, gpt2]) data = spark.createDataFrame([["My name is Leonardo."]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("summaries.generation").show(truncate=False) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val gpt2 = GPT2Transformer.pretrained("gpt2_distilled") .setInputCols(Array("documents")) .setMinOutputLength(10) .setMaxOutputLength(50) .setDoSample(false) .setTopK(50) .setNoRepeatNgramSize(3) .setOutputCol("generation") val pipeline = new Pipeline().setStages(Array(documentAssembler, gpt2)) val data = Seq("My name is Leonardo.").toDF("text") val result = pipeline.fit(data).transform(data) results.select("generation.result").show(truncate = false) ``` {:.nlu-block} ```python import nlu nlu.load("en.gpt2.distilled").predict("""My name is Leonardo.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|gpt2_distilled| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[generation]| |Language:|en| ## Data Source OpenAI WebText - a corpus created by scraping web pages with emphasis on document quality. It consists of over 8 million documents for a total of 40 GB of text. All Wikipedia documents were removed from WebText. --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_mini_finetuned_squadv2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-mini-finetuned-squadv2` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_mini_finetuned_squadv2_en_4.0.0_3.0_1654183774865.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_mini_finetuned_squadv2_en_4.0.0_3.0_1654183774865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_mini_finetuned_squadv2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_mini_finetuned_squadv2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.base_v2.by_mrm8488").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_mini_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|42.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-mini-finetuned-squadv2 - https://twitter.com/mrm8488 - https://github.com/google-research - https://arxiv.org/abs/1908.08962 - https://rajpurkar.github.io/SQuAD-explorer/ - https://github.com/google-research/bert/ - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: German asr_exp_w2v2t_wav2vec2_s982 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2t_wav2vec2_s982 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_wav2vec2_s982` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2t_wav2vec2_s982_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_wav2vec2_s982_de_4.2.0_3.0_1664122237111.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_wav2vec2_s982_de_4.2.0_3.0_1664122237111.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2t_wav2vec2_s982", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2t_wav2vec2_s982", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2t_wav2vec2_s982| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: French CamemBert Embeddings (from Hasanmuradbuet) author: John Snow Labs name: camembert_embeddings_Hasanmuradbuet_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `Hasanmuradbuet`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Hasanmuradbuet_generic_model_fr_3.4.4_3.0_1653986214124.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_Hasanmuradbuet_generic_model_fr_3.4.4_3.0_1653986214124.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_Hasanmuradbuet_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/Hasanmuradbuet/dummy-model --- layout: model title: Legal Non Soliciting Clause Binary Classifier author: John Snow Labs name: legclf_non_solic_clause date: 2023-02-13 tags: [en, legal, classification, non_soliciting, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `non_solic` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `non_solic`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_non_solic_clause_en_1.0.0_3.0_1676305039177.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_non_solic_clause_en_1.0.0_3.0_1676305039177.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_non_solic_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[non_solic]| |[other]| |[other]| |[non_solic]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_non_solic_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support non_solic 0.92 1.00 0.96 11 other 1.00 0.80 0.89 5 accuracy - - 0.94 16 macro-avg 0.96 0.90 0.92 16 weighted-avg 0.94 0.94 0.94 16 ``` --- layout: model title: Multilingual XLMRobertaForTokenClassification Cased model (from jplu) author: John Snow Labs name: xlmroberta_ner_tf_r_40_lang date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tf-xlm-r-ner-40-lang` is a Multilingual model originally trained by `jplu`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tf_r_40_lang_xx_4.1.0_3.0_1660423122270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tf_r_40_lang_xx_4.1.0_3.0_1660423122270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tf_r_40_lang","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tf_r_40_lang","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tf_r_40_lang| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|967.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/jplu/tf-xlm-r-ner-40-lang - https://arxiv.org/abs/1911.02116 - https://github.com/google-research/xtreme - https://aclweb.org/anthology/P17-1178 - https://github.com/google-research/xtreme#download-the-data --- layout: model title: NER Model for 10 African Languages author: John Snow Labs name: xlm_roberta_large_token_classifier_masakhaner date: 2021-12-06 tags: [amharic, hausa, igbo, kinyarwanda, luganda, swahilu, wolof, yoruba, token_classifier, xlm_roberta, ner, nigerian, pidgin, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 3.3.2 spark_version: 2.4 supported: true recommended: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description - This model is imported from `Hugging Face`. - It's been trained using `xlm_roberta_large` fine-tuned model on 10 African languages (Amharic, Hausa, Igbo, Kinyarwanda, Luganda, Nigerian, Pidgin, Swahilu, Wolof, and Yorùbá). ## Predicted Entities `DATE`, `LOC`, `PER`, `ORG` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/Ner_masakhaner/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Ner_masakhaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_masakhaner_xx_3.3.2_2.4_1638784947143.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_large_token_classifier_masakhaner_xx_3.3.2_2.4_1638784947143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------+---------+ |chunk |ner_label| +----------------+---------+ |አህመድ ቫንዳ |PER | |ከ3-10-2000 ጀምሮ|DATE | |በአዲስ አበባ |LOC | +----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_large_token_classifier_masakhaner| |Compatibility:|Spark NLP 3.3.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|xx| |Case sensitive:|true| |Max sentense length:|256| ## Data Source [https://huggingface.co/Davlan/xlm-roberta-large-masakhaner](https://huggingface.co/Davlan/xlm-roberta-large-masakhaner) ## Benchmarking ```bash language: F1-score: -------- -------- amh 75.76 hau 91.75 ibo 86.26 kin 76.38 lug 84.64 luo 80.65 pcm 89.55 swa 89.48 wol 70.70 yor 82.05 ``` --- layout: model title: Detect Clinical Entities (clinical_medium) author: John Snow Labs name: ner_jsl_emb_clinical_medium date: 2023-04-12 tags: [ner, licensed, clinical, en, clinical_medium] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state-of-the-art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. This model is the official version of jsl_ner_wip_clinical model. Definitions of Predicted Entities: - `Injury_or_Poisoning`: Physical harm or injury caused to the body, including those caused by accidents, falls, or poisoning of a patient or someone else. - `Direction`: All the information relating to the laterality of the internal and external organs. - `Test`: Mentions of laboratory, pathology, and radiological tests. - `Admission_Discharge`: Terms that indicate the admission and/or the discharge of a patient. - `Death_Entity`: Mentions that indicate the death of a patient. - `Relationship_Status`: State of patients romantic or social relationships (e.g. single, married, divorced). - `Duration`: The duration of a medical treatment or medication use. - `Respiration`: Number of breaths per minute. - `Hyperlipidemia`: Terms that indicate hyperlipidemia with relevant subtypes and synonims. - `Birth_Entity`: Mentions that indicate giving birth. - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Labour_Delivery`: Extractions include stages of labor and delivery. - `Family_History_Header`: identifies section headers that correspond to Family History of the patient. - `BMI`: Numeric values and other text information related to Body Mass Index. - `Temperature`: All mentions that refer to body temperature. - `Alcohol`: Terms that indicate alcohol use, abuse or drinking issues of a patient or someone else. - `Kidney_Disease`: Terms that refer to any kidney diseases (includes mentions of modifiers such as "Acute" or "Chronic"). - `Oncological`: All the cancer, tumor or metastasis related extractions mentioned in the document, of the patient or someone else. - `Medical_History_Header`: Identifies section headers that correspond to Past Medical History of a patient. - `Cerebrovascular_Disease`: All terms that refer to cerebrovascular diseases and events. - `Oxygen_Therapy`: Breathing support triggered by patient or entirely or partially by machine (e.g. ventilator, BPAP, CPAP). - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Psychological_Condition`: All the Mental health diagnosis, disorders, conditions or syndromes of a patient or someone else. - `Heart_Disease`: All mentions of acquired, congenital or degenerative heart diseases. - `Employment`: All mentions of patient or provider occupational titles and employment status . - `Obesity`: Terms related to a patient being obese (overweight and BMI are extracted as different labels). - `Disease_Syndrome_Disorder`: All the diseases mentioned in the document, of the patient or someone else (excluding diseases that are extracted with their specific labels, such as "Heart_Disease" etc.). - `Pregnancy`: All terms related to Pregnancy (excluding terms that are extracted with their specific labels, such as "Labour_Delivery" etc.). - `ImagingFindings`: All mentions of radiographic and imagistic findings. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Medical_Device`: All mentions related to medical devices and supplies. - `Race_Ethnicity`: All terms that refer to racial and national origin of sociocultural groups. - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Treatment`: Includes therapeutic and minimally invasive treatment and procedures (invasive treatments or procedures are extracted as "Procedure"). - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_Ingredient`: Active ingredient/s found in drug products. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Diet`: All mentions and information regarding patients dietary habits. - `External_body_part_or_region`: All mentions related to external body parts or organs that can be examined by naked eye. - `LDL`: All mentions related to the lab test and results for LDL (Low Density Lipoprotein). - `VS_Finding`: Qualitative data (e.g. Fever, Cyanosis, Tachycardia) and any other symptoms that refers to vital signs. - `Allergen`: Allergen related extractions mentioned in the document. - `EKG_Findings`: All mentions of EKG readings. - `Imaging_Technique`: All mentions of special radiographic views or special imaging techniques used in radiology. - `Triglycerides`: All mentions terms related to specific lab test for Triglycerides. - `RelativeTime`: Time references that are relative to different times or events (e.g. words such as "approximately", "in the morning"). - `Gender`: Gender-specific nouns and pronouns. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Social_History_Header`: Identifies section headers that correspond to Social History of a patient. - `Substance_Quantity`: All mentions of substance quantity (quantitative information related to illicit/recreational drugs). - `Diabetes`: All terms related to diabetes mellitus. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `Internal_organ_or_component`: All mentions related to internal body parts or organs that can not be examined by naked eye. - `Clinical_Dept`: Terms that indicate the medical and/or surgical departments. - `Form`: Drug and medication forms available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_BrandName`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Strength`: Potency of one unit of drug (or a combination of drugs) the measurement units available are described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Fetus_NewBorn`: All terms related to fetus, infant, new born (excluding terms that are extracted with their specific labels, such as "Labour_Delivery", "Pregnancy" etc.). - `RelativeDate`: Temporal references that are relative to the date of the text or to any other specific date (e.g. "approximately two years ago", "about two days ago"). - `Height`: All mentions related to a patients height. - `Test_Result`: Terms related to all the test results present in the document (clinical tests results are included). - `Sexually_Active_or_Sexual_Orientation`: All terms that are related to sexuality, sexual orientations and sexual activity. - `Frequency`: Frequency of administration for a dose prescribed. - `Time`: Specific time references (hour and/or minutes). - `Weight`: All mentions related to a patients weight. - `Vaccine`: Generic and brand name of vaccines or vaccination procedure. - `Vital_Signs_Header`: Identifies section headers that correspond to Vital Signs of a patient. - `Communicable_Disease`: Includes all mentions of communicable diseases. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Overweight`: Terms related to the patient being overweight (BMI and Obesity is extracted separately). - `Hypertension`: All terms related to Hypertension (quantitative data such as 150/100 is extracted as Blood_Pressure). - `HDL`: Terms related to the lab test for HDL (High Density Lipoprotein). - `Total_Cholesterol`: Terms related to the lab test and results for cholesterol. - `Smoking`: All mentions of smoking status of a patient. - `Date`: Mentions of an exact date, in any format, including day number, month and/or year. ## Predicted Entities `Injury_or_Poisoning`, `Direction`, `Test`, `Admission_Discharge`, `Death_Entity`, `Relationship_Status`, `Duration`, `Respiration`, `Hyperlipidemia`, `Birth_Entity`, `Age`, `Labour_Delivery`, `Family_History_Header`, `BMI`, `Temperature`, `Alcohol`, `Kidney_Disease`, `Oncological`, `Medical_History_Header`, `Cerebrovascular_Disease`, `Oxygen_Therapy`, `O2_Saturation`, `Psychological_Condition`, `Heart_Disease`, `Employment`, `Obesity`, `Disease_Syndrome_Disorder`, `Pregnancy`, `ImagingFindings`, `Procedure`, `Medical_Device`, `Race_Ethnicity`, `Section_Header`, `Symptom`, `Treatment`, `Substance`, `Route`, `Drug_Ingredient`, `Blood_Pressure`, `Diet`, `External_body_part_or_region`, `LDL`, `VS_Finding`, `Allergen`, `EKG_Findings`, `Imaging_Technique`, `Triglycerides`, `RelativeTime`, `Gender`, `Pulse`, `Social_History_Header`, `Substance_Quantity`, `Diabetes`, `Modifier`, `Internal_organ_or_component`, `Clinical_Dept`, `Form`, `Drug_BrandName`, `Strength`, `Fetus_NewBorn`, `RelativeDate`, `Height`, `Test_Result`, `Sexually_Active_or_Sexual_Orientation`, `Frequency`, `Time`, `Weight`, `Vaccine`, `Vaccine_Name`, `Vital_Signs_Header`, `Communicable_Disease`, `Dosage`, `Overweight`, `Hypertension`, `HDL`, `Total_Cholesterol`, `Smoking`, `Date` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_emb_clinical_medium_en_4.3.2_3.0_1681306334405.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_emb_clinical_medium_en_4.3.2_3.0_1681306334405.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------------------------+-----+---+----------------------------+ |chunk |begin|end|ner_label | +-----------------------------------------+-----+---+----------------------------+ |21-day-old |17 |26 |Age | |Caucasian |28 |36 |Race_Ethnicity | |male |38 |41 |Gender | |2 days |52 |57 |Duration | |congestion |62 |71 |Symptom | |mom |75 |77 |Gender | |suctioning yellow discharge |88 |114|Symptom | |nares |135 |139|External_body_part_or_region| |she |147 |149|Gender | |mild |168 |171|Modifier | |problems with his breathing while feeding|173 |213|Symptom | |perioral cyanosis |237 |253|Symptom | |retractions |258 |268|Symptom | |Influenza vaccine |325 |341|Vaccine_Name | |One day ago |344 |354|RelativeDate | |mom |357 |359|Gender | |tactile temperature |376 |394|Symptom | |Tylenol |417 |423|Drug_BrandName | |Baby |426 |429|Age | |decreased p.o |449 |461|Symptom | +-----------------------------------------+-----+---+----------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.2 MB| ## Benchmarking ```bash label precision recall f1-score support Drug_Ingredient 0.91 0.94 0.93 1905 Disease_Syndrome_Disorder 0.85 0.89 0.87 4949 Drug_BrandName 0.94 0.92 0.93 963 Strength 0.95 0.93 0.94 759 Route 0.90 0.94 0.92 943 Internal_organ_or_component 0.89 0.90 0.89 10310 Dosage 0.95 0.78 0.86 478 Frequency 0.90 0.87 0.88 1016 Treatment 0.90 0.70 0.78 332 Procedure 0.85 0.91 0.88 6433 Gender 0.98 0.99 0.99 5586 RelativeTime 0.79 0.70 0.74 306 Direction 0.91 0.90 0.91 4344 Modifier 0.84 0.82 0.83 2863 Symptom 0.84 0.83 0.84 11599 Date 0.94 0.98 0.96 546 External_body_part_or_region 0.89 0.85 0.87 3270 Section_Header 0.98 0.97 0.98 9320 Age 0.85 0.92 0.88 757 Substance 0.91 0.85 0.88 113 VS_Finding 0.86 0.60 0.70 304 Medical_Device 0.87 0.93 0.90 5475 Oxygen_Therapy 0.89 0.85 0.87 117 Test 0.87 0.87 0.87 4491 Diabetes 0.95 0.97 0.96 149 Duration 0.86 0.86 0.86 1009 ImagingFindings 0.83 0.50 0.63 353 Hyperlipidemia 0.80 0.87 0.84 47 Hypertension 0.97 0.94 0.96 152 RelativeDate 0.90 0.89 0.89 1338 Clinical_Dept 0.92 0.94 0.93 1771 Kidney_Disease 0.90 0.96 0.93 228 Heart_Disease 0.93 0.85 0.89 967 Diet 0.67 0.62 0.65 106 Weight 0.93 0.93 0.93 254 Test_Result 0.79 0.81 0.80 1470 Form 0.85 0.85 0.85 254 Time 0.80 0.72 0.76 76 Psychological_Condition 0.81 0.76 0.79 187 Injury_or_Poisoning 0.84 0.80 0.82 889 Admission_Discharge 0.91 0.97 0.94 301 Labour_Delivery 0.74 0.73 0.73 110 Employment 0.90 0.73 0.81 389 Vaccine 0.83 0.33 0.48 15 Obesity 0.92 0.91 0.92 54 Oncological 0.92 0.93 0.93 784 Smoking 0.96 0.94 0.95 106 Imaging_Technique 0.70 0.53 0.60 98 Blood_Pressure 0.86 0.87 0.86 314 Pulse 0.87 0.92 0.89 278 Respiration 0.96 0.94 0.95 180 O2_Saturation 0.86 0.77 0.81 96 Medical_History_Header 0.93 0.99 0.96 396 Total_Cholesterol 0.88 0.37 0.52 19 Cerebrovascular_Disease 0.75 0.78 0.76 108 Pregnancy 0.90 0.71 0.79 201 Death_Entity 0.85 0.76 0.80 46 EKG_Findings 0.78 0.43 0.56 186 Race_Ethnicity 0.97 0.99 0.98 118 Family_History_Header 0.97 0.99 0.98 273 Alcohol 0.84 0.95 0.89 84 Fetus_NewBorn 0.75 0.54 0.63 235 Vital_Signs_Header 0.96 0.94 0.95 710 Relationship_Status 0.93 0.95 0.94 41 Height 0.98 0.85 0.91 68 Temperature 0.91 0.96 0.94 141 Triglycerides 0.40 0.40 0.40 10 LDL 0.94 0.68 0.79 22 Social_History_Header 0.98 0.97 0.98 259 Communicable_Disease 0.79 0.62 0.70 50 Overweight 0.86 0.86 0.86 7 Allergen 0.00 0.00 0.00 23 Substance_Quantity 0.33 1.00 0.50 2 Vaccine_Name 0.68 1.00 0.81 15 Birth_Entity 0.00 0.00 0.00 3 BMI 1.00 0.67 0.80 15 Sexually_Active_or_Sexual_Orientation 1.00 0.80 0.89 5 HDL 1.00 1.00 1.00 2 micro-avg 0.89 0.89 0.89 92193 macro-avg 0.85 0.81 0.82 92193 weighted-avg 0.89 0.89 0.89 92193 ``` --- layout: model title: Detect Radiology Related Entities author: John Snow Labs name: ner_radiology date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for radiology related texts and reports. ## Predicted Entities `ImagingTest`, `Imaging_Technique`, `ImagingFindings`, `OtherFindings`, `BodyPart`, `Direction`, `Test`, `Symptom`, `Disease_Syndrome_Disorder`, `Medical_Device`, `Procedure`, `Measurements`, `Units` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_RADIOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_radiology_en_3.0.0_3.0_1617208452337.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_radiology_en_3.0.0_3.0_1617208452337.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") radiology_ner = MedicalNerModel.pretrained("ner_radiology", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, radiology_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma."]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val radiology_ner = MedicalNerModel().pretrained("ner_radiology", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val nlpPipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, radiology_ner, ner_converter)) val data = Seq("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""").toDS.toDF("text") val result = nlpPipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.radiology").predict("""Bilateral breast ultrasound was subsequently performed, which demonstrated an ovoid mass measuring approximately 0.5 x 0.5 x 0.4 cm in diameter located within the anteromedial aspect of the left shoulder. This mass demonstrates isoechoic echotexture to the adjacent muscle, with no evidence of internal color flow. This may represent benign fibrous tissue or a lipoma.""") ```

## Results ```bash | | chunks | entities | |----|-----------------------|---------------------------| | 0 | Bilateral | Direction | | 1 | breast | BodyPart | | 2 | ultrasound | ImagingTest | | 3 | ovoid mass | ImagingFindings | | 4 | 0.5 x 0.5 x 0.4 | Measurements | | 5 | cm | Units | | 6 | anteromedial aspect | Direction | | 7 | left | Direction | | 8 | shoulder | BodyPart | | 9 | mass | ImagingFindings | | 10 | isoechoic echotexture | ImagingFindings | | 11 | muscle | BodyPart | | 12 | internal color flow | ImagingFindings | | 13 | benign fibrous tissue | ImagingFindings | | 14 | lipoma | Disease_Syndrome_Disorder | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_radiology| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on a custom dataset comprising of MIMIC-CXR and MT Radiology texts ## Benchmarking ```bash +--------------------+------+-----+-----+------+---------+------+------+ | entity| tp| fp| fn| total|precision|recall| f1| +--------------------+------+-----+-----+------+---------+------+------+ | OtherFindings| 8.0| 15.0| 63.0| 71.0| 0.3478|0.1127|0.1702| | Measurements| 481.0| 30.0| 15.0| 496.0| 0.9413|0.9698|0.9553| | Direction| 650.0|137.0| 94.0| 744.0| 0.8259|0.8737|0.8491| | ImagingFindings|1345.0|355.0|324.0|1669.0| 0.7912|0.8059|0.7985| | BodyPart|1942.0|335.0|290.0|2232.0| 0.8529|0.8701|0.8614| | Medical_Device| 236.0| 75.0| 64.0| 300.0| 0.7588|0.7867|0.7725| | Test| 222.0| 41.0| 48.0| 270.0| 0.8441|0.8222| 0.833| | Procedure| 269.0|117.0|116.0| 385.0| 0.6969|0.6987|0.6978| | ImagingTest| 263.0| 50.0| 43.0| 306.0| 0.8403|0.8595|0.8498| | Symptom| 498.0|101.0|132.0| 630.0| 0.8314|0.7905|0.8104| |Disease_Syndrome_...|1180.0|258.0|200.0|1380.0| 0.8206|0.8551|0.8375| | Units| 269.0| 10.0| 2.0| 271.0| 0.9642|0.9926|0.9782| | Imaging_Technique| 140.0| 38.0| 25.0| 165.0| 0.7865|0.8485|0.8163| +--------------------+------+-----+-----+------+---------+------+------+ +------------------+ | macro| +------------------+ |0.7524248724038437| +------------------+ +------------------+ | micro| +------------------+ |0.8315240382681794| +------------------+ ``` --- layout: model title: Swedish BertForTokenClassification Base Cased model (from KBLab) author: John Snow Labs name: bert_token_classifier_base_swedish_lowermix_reallysimple_ner date: 2022-11-30 tags: [sv, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: sv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-swedish-lowermix-reallysimple-ner` is a Swedish model originally trained by `KBLab`. ## Predicted Entities `TME`, `OBJ`, `MSR`, `LOC`, `ORG`, `PRS`, `WRK`, `EVN` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_swedish_lowermix_reallysimple_ner_sv_4.2.4_3.0_1669815085727.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_base_swedish_lowermix_reallysimple_ner_sv_4.2.4_3.0_1669815085727.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_swedish_lowermix_reallysimple_ner","sv") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_base_swedish_lowermix_reallysimple_ner","sv") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_base_swedish_lowermix_reallysimple_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|sv| |Size:|465.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/KBLab/bert-base-swedish-lowermix-reallysimple-ner - https://kb-labb.github.io/posts/2022-02-07-sucx3_ner --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl20 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl20` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl20_en_4.3.0_3.0_1675121765443.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl20_en_4.3.0_3.0_1675121765443.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl20","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl20","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl20| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|345.3 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl20 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Explain Document Pipeline for Spanish author: John Snow Labs name: explain_document_sm date: 2021-03-22 tags: [open_source, spanish, explain_document_sm, pipeline, es] supported: true task: [Named Entity Recognition, Lemmatization, Part of Speech Tagging] language: es edition: Spark NLP 3.0.0 spark_version: 3.0 article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_sm is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/jupyter/annotation/english/explain-document-dl/Explain%20Document%20DL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_sm_es_3.0.0_3.0_1616422359763.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_sm_es_3.0.0_3.0_1616422359763.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('explain_document_sm', lang = 'es') annotations = pipeline.fullAnnotate(""Hola de John Snow Labs! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_sm", lang = "es") val result = pipeline.fullAnnotate("Hola de John Snow Labs! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hola de John Snow Labs! ""] result_df = nlu.load('es.explain').predict(text) result_df ```

## Results ```bash | | document | sentence | token | lemma | pos | embeddings | ner | entities | |---:|:-----------------------------|:----------------------------|:----------------------------------------|:----------------------------------------|:-------------------------------------------|:-----------------------------|:---------------------------------------|:-----------------------| | 0 | ['Hola de John Snow Labs! '] | ['Hola de John Snow Labs!'] | ['Hola', 'de', 'John', 'Snow', 'Labs!'] | ['Hola', 'de', 'John', 'Snow', 'Labs!'] | ['PART', 'ADP', 'PROPN', 'PROPN', 'PROPN'] | [[0.1754499971866607,.,...]] | ['O', 'O', 'B-PER', 'I-PER', 'B-MISC'] | ['John Snow', 'Labs!'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_sm| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|es| --- layout: model title: Pipeline to Detect Anatomical Structures in Medical Text author: John Snow Labs name: bert_token_classifier_ner_anatem_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_anatem](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_anatem_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatem_pipeline_en_4.3.0_3.2_1679303142191.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_anatem_pipeline_en_4.3.0_3.2_1679303142191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-----------------------|--------:|------:|:------------|-------------:| | 0 | Malignant cells | 0 | 14 | Anatomy | 0.999951 | | 1 | cytoplasmic organelles | 126 | 147 | Anatomy | 0.999937 | | 2 | tumor | 229 | 233 | Anatomy | 0.999871 | | 3 | breast | 343 | 348 | Anatomy | 0.999842 | | 4 | ovarian | 351 | 357 | Anatomy | 0.99998 | | 5 | prostate cancer | 364 | 378 | Anatomy | 0.999968 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_anatem_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: English asr_wav2vec2_base_timit_demo_colab2_by_hassnain TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab2_by_hassnain date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab2_by_hassnain` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab2_by_hassnain_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_hassnain_en_4.2.0_3.0_1664039971215.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_hassnain_en_4.2.0_3.0_1664039971215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_hassnain", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_hassnain", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab2_by_hassnain| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Icelandic Lemmatizer author: John Snow Labs name: lemma date: 2021-04-02 tags: [is, open_source, lemmatizer] task: Lemmatization language: is edition: Spark NLP 2.7.5 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a dictionary-based lemmatizer that assigns all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_is_2.7.5_2.4_1617376506935.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_is_2.7.5_2.4_1617376506935.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------+ | lemma| +-----------+ | hinn| | þar| | vera| | þeir| | vinnast| | ekki| | til| | þá| | hafa| | þeir| | ég| | aðra| | stafur| | svo| | margur| |og-og-og-og| | þesskonar| | sem| | þurfa| | hinn| +-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|is| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) version 2.7. ## Benchmarking ```bash Precision=0.63, Recall=0.59, F1-score=0.61 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from deep-learning-analytics) author: John Snow Labs name: t5_triviaqa_base date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `triviaqa-t5-base` is a English model originally trained by `deep-learning-analytics`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_triviaqa_base_en_4.3.0_3.0_1675157650979.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_triviaqa_base_en_4.3.0_3.0_1675157650979.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_triviaqa_base","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_triviaqa_base","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_triviaqa_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|927.9 MB| ## References - https://huggingface.co/deep-learning-analytics/triviaqa-t5-base - https://medium.com/@priya.dwivedi/build-a-trivia-bot-using-t5-transformer-345ff83205b6 - https://www.triviaquestionss.com/easy-trivia-questions/ - https://laffgaff.com/easy-trivia-questions-and-answers/ --- layout: model title: English DistilBertForQuestionAnswering Cased model (from jadegao) author: John Snow Labs name: distilbert_qa_askinvesto_model date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `askinvesto-distilbert-model` is a English model originally trained by `jadegao`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_askinvesto_model_en_4.3.0_3.0_1672765610636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_askinvesto_model_en_4.3.0_3.0_1672765610636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_askinvesto_model","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_askinvesto_model","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_askinvesto_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jadegao/askinvesto-distilbert-model --- layout: model title: ICD10 to UMLS Code Mapping author: John Snow Labs name: icd10cm_umls_mapping date: 2021-07-01 tags: [icd10cm, umls, en, licensed, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps ICD10CM codes to UMLS codes without using any text data. You’ll just feed white space-delimited ICD10CM codes and it will return the corresponding UMLS codes as a list. If there is no mapping, the original code is returned with no mapping. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_CODE_MAPPING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_3.1.0_2.4_1625126281405.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_3.1.0_2.4_1625126281405.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline( "icd10cm_umls_mapping","en","clinical/models") pipeline.annotate(["M8950", "R822", "R0901"]) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10cm_umls_mapping","en","clinical/models") val result = pipeline.annotate("M8950", "R822", "R0901") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.icd10cm.umls").predict("""M8950 R822 R0901""") ```

## Results ```bash {'icd10cm': ['M89.50', 'R82.2', 'R09.01'], 'umls': ['C4721411', 'C0159076', 'C0004044']} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - TokenizerModel - LemmatizerModel - Finisher --- layout: model title: English asr_wav2vec2_large_xls_r_300m_georgian_v0.6 TFWav2Vec2ForCTC from pavle-tsotskolauri author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_georgian_v0.6` is a English model originally trained by pavle-tsotskolauri. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6_en_4.2.0_3.0_1664119076573.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6_en_4.2.0_3.0_1664119076573.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_georgian_v0.6| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|2.3 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Mapping MESH Codes with Their Corresponding UMLS Codes author: John Snow Labs name: mesh_umls_mapper date: 2022-06-26 tags: [mesh, umls, chunk_mapper, clinical, licensed, en] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps MESH codes to corresponding UMLS codes under the Unified Medical Language System (UMLS). ## Predicted Entities `umls_code` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapper_en_3.5.3_3.0_1656281333787.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/mesh_umls_mapper_en_3.5.3_3.0_1656281333787.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli", "en", "clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings")\ .setCaseSensitive(False) mesh_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_mesh", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("mesh_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel\ .pretrained("mesh_umls_mapper", "en", "clinical/models")\ .setInputCols(["mesh_code"])\ .setOutputCol("umls_mappings")\ .setRels(["umls_code"]) pipeline = Pipeline(stages = [ documentAssembler, sbert_embedder, mesh_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) light_pipeline= LightPipeline(model) result = light_pipeline.fullAnnotate("N-acetyl-L-arginine") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols("ner_chunk") .setOutputCol("sbert_embeddings") val mesh_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_mesh", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("mesh_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel .pretrained("mesh_umls_mapper", "en", "clinical/models") .setInputCols("mesh_code") .setOutputCol("umls_mappings") .setRels(Array("umls_code")) val pipeline = new Pipeline(stages = Array( documentAssembler, sbert_embedder, mesh_resolver, chunkerMapper )) val data = Seq("N-acetyl-L-arginine").toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.mesh_to_umls").predict("""Put your text here.""") ```

## Results ```bash | | ner_chunk | mesh_code | umls_mappings | |---:|:--------------------|:------------|:----------------| | 0 | N-acetyl-L-arginine | C000015 | C0067655 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|mesh_umls_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[mesh_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|3.8 MB| --- layout: model title: English asr_wav2vec2_large_960h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_large_960h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_960h` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_960h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_en_4.2.0_3.0_1664016613522.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_960h_en_4.2.0_3.0_1664016613522.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_960h', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_960h", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_960h| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|755.4 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-128-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10_en_4.0.0_3.0_1655731124938.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10_en_4.0.0_3.0_1655731124938.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_128d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_128_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|422.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-128-finetuned-squad-seed-10 --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_nl8 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-nl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl8_en_4.3.0_3.0_1675114011208.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_nl8_en_4.3.0_3.0_1675114011208.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_nl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_nl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_nl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|348.0 MB| ## References - https://huggingface.co/google/t5-efficient-base-nl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Fast Neural Machine Translation Model from Afro-Asiatic languages to Afro-Asiatic languages author: John Snow Labs name: opus_mt_afa_afa date: 2021-06-01 tags: [open_source, seq2seq, translation, afa, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: afa target languages: afa {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_afa_afa_xx_3.1.0_2.4_1622553414579.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_afa_afa_xx_3.1.0_2.4_1622553414579.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_afa_afa| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada TFWav2Vec2ForCTC from nadaAlnada author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada` is a English model originally trained by nadaAlnada. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada_en_4.2.0_3.0_1664114063867.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada_en_4.2.0_3.0_1664114063867.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_nadaAlnada| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_rxnorm date: 2021-10-10 tags: [rxnorm, entity_resolution, licensed, clinical, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.2.3 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to RxNorm codes using `sbiobert_base_cased_mli ` Sentence Bert Embeddings. ## Predicted Entities `RxNorm Codes` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_RXNORM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_3.2.3_2.4_1633875017884.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_en_3.2.3_2.4_1633875017884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------+------+----------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk|entity|icd10_code| all_codes| resolutions| +-------------+------+----------+----------------------------------------------------------------------+----------------------------------------------------------------------+ | Fragmin| DRUG| 281554|281554:::1992532:::151794:::217814:::361779:::1098701:::203870:::15...|Fragmin:::Kedrab:::Frisium:::Isopto Frin:::Frumax:::Folgard:::Faslo...| | Xenaderm| DRUG| 581754|581754:::898496:::202363:::1307304:::94611:::1046399:::1093360:::11...|Xenaderm:::Xiaflex:::Xanax:::Xtandi:::Xerac AC:::Xgeva:::Xoten:::Xa...| | lantus| DRUG| 261551|261551:::151959:::28381:::202990:::196502:::608814:::1040032:::7049...|Lantus:::Laratrim:::lachesine:::Larodopa:::Lamictal:::Lansinoh:::La...| | OxyContin| DRUG| 218986|218986:::32680:::32698:::352843:::218859:::1086614:::1120014:::2189...|Oxycontin:::oxychlorosene:::oxyphencyclimine:::Ocutricin HC:::Ocutr...| | folic acid| DRUG| 4511|4511:::1162058:::1162059:::62356:::1376005:::542060:::619039:::1162...|folic acid:::folic acid Oral Product:::folic acid Pill:::folate:::F...| |levothyroxine| DRUG| 10582|10582:::1868004:::40144:::1602753:::1602745:::227577:::1602750:::11...|levothyroxine:::levothyroxine Injection:::levothyroxine sodium:::le...| | Prevacid| DRUG| 83156|83156:::219485:::219171:::606051:::858359:::1547099:::2286610:::105...|Prevacid:::Provisc:::Perisine:::ProQuad:::Acuvail:::suvorexant:::Pi...| | Avandia| DRUG| 261455|261455:::152800:::1310526:::236219:::1370666:::686438:::215221:::99...|Avandia:::Amilamont:::Aubagio:::alibendol:::anisate:::Invega:::Amil...| | norvasc| DRUG| 58927|58927:::1876388:::218772:::262324:::385700:::226108:::203013:::2036...|Norvasc:::NoRisc:::Norco:::Norflex:::Norval:::Norimode:::Norcuron::...| | lexapro| DRUG| 352741|352741:::580253:::227285:::2058916:::24867:::847463:::2055761:::144...|Lexapro:::Levsinex:::Loprox:::Vizimpro:::fenproporex:::Levoprome:::...| | aspirin| DRUG| 1191|1191:::405403:::218266:::1154070:::215568:::202547:::1154069:::2393...|aspirin:::YSP Aspirin:::Med Aspirin:::aspirin Pill:::Bayer Aspirin:...| | Neurontin| DRUG| 196498|196498:::152627:::151178:::827343:::134802:::1720602:::152141:::131...|Neurontin:::Nystamont:::Nitronal:::Nucort:::Naropin:::Nucala:::Nyst...| +-------------+------+----------+----------------------------------------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm| |Compatibility:|Healthcare NLP 3.2.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on 02 August 2021 RxNorm dataset. --- layout: model title: English DistilBertForQuestionAnswering model (from mcurmei) Single author: John Snow Labs name: distilbert_qa_single_label_N_max date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `single_label_N_max` is a English model originally trained by `mcurmei`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_single_label_N_max_en_4.0.0_3.0_1654728665750.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_single_label_N_max_en_4.0.0_3.0_1654728665750.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_single_label_N_max","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_single_label_N_max","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.single_label_n_max.by_mcurmei").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_single_label_N_max| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mcurmei/single_label_N_max --- layout: model title: Extract Mentions of Response to Cancer Treatment author: John Snow Labs name: ner_oncology_response_to_treatment date: 2022-10-25 tags: [licensed, clinical, oncology, en, ner, treatment] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP for Healthcare 4.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to the patient"s response to the oncology treatment, including clinical response and changes in tumor size. Definitions of Predicted Entities: - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Size_Trend`: Terms related to the changes in the size of the tumor (such as "growth" or "reduced in size"). ## Predicted Entities `Line_Of_Therapy`, `Response_To_Treatment`, `Size_Trend` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_en_4.0.0_3.0_1666722959227.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_en_4.0.0_3.0_1666722959227.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------|:----------------------| | first-line | Line_Of_Therapy | | recurrence | Response_To_Treatment | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_response_to_treatment| |Compatibility:|Spark NLP for Healthcare 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.4 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Response_To_Treatment 326 101 157 483 0.76 0.67 0.72 Size_Trend 43 28 70 113 0.61 0.38 0.47 Line_Of_Therapy 99 11 7 106 0.90 0.93 0.92 macro_avg 468 140 234 702 0.76 0.66 0.70 micro_avg 468 140 234 702 0.76 0.67 0.71 ``` --- layout: model title: Persian Lemmatizer author: John Snow Labs name: lemma date: 2020-11-28 task: Lemmatization language: fa edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [lemmatizer, fa, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_fa_2.7.0_2.4_1606581127793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_fa_2.7.0_2.4_1606581127793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of a pipeline after tokenisation.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "fa") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate(['فرماندهان فرماندهی فرمانده فرمانده‌ای فرماندهٔ فرماند']) ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "fa") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("فرماندهان فرماندهی فرمانده فرمانده‌ای فرماندهٔ فرماند").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""فرماندهان فرماندهی فرمانده فرمانده‌ای فرماندهٔ فرماند"""] lemma_df = nlu.load('fa.lemma').predict(text) lemma_df.lemma.values[0] ```

## Results ```bash {'lemma': [Annotation(token, 0, 8, فرمانده, {'sentence': '0'}), Annotation(token, 10, 17, فرماندهی, {'sentence': '0'}), Annotation(token, 19, 25, فرمانده, {'sentence': '0'}), Annotation(token, 27, 36, فرمانده, {'sentence': '0'}), Annotation(token, 38, 45, فرمانده, {'sentence': '0'}), Annotation(token, 47, 52, فرماند, {'sentence': '0'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|fa| ## Data Source This model is trained on data obtained from [https://universaldependencies.org/](https://universaldependencies.org/) --- layout: model title: Dutch Named Entity Recognition (from gunghio) author: John Snow Labs name: distilbert_ner_distilbert_base_multilingual_cased_finetuned_conll2003_ner date: 2022-05-16 tags: [distilbert, ner, token_classification, nl, open_source] task: Named Entity Recognition language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multilingual-cased-finetuned-conll2003-ner` is a Dutch model orginally trained by `gunghio`. ## Predicted Entities `ORG`, `MISC`, `PER`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_multilingual_cased_finetuned_conll2003_ner_nl_3.4.2_3.0_1652721741727.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_multilingual_cased_finetuned_conll2003_ner_nl_3.4.2_3.0_1652721741727.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_distilbert_base_multilingual_cased_finetuned_conll2003_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|nl| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner - https://paperswithcode.com/sota?task=Named+Entity+Recognition&dataset=ConLL+2003 --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nl8 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nl8` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl8_en_4.3.0_3.0_1675123995941.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl8_en_4.3.0_3.0_1675123995941.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nl8","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nl8","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nl8| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|60.3 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nl8 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Multilingual T5ForConditionalGeneration Small Cased model (from north) author: John Snow Labs name: t5_small_ncc date: 2023-01-31 tags: [is, nn, en, "no", sv, open_source, t5, xx, tensorflow] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5_small_NCC` is a Multilingual model originally trained by `north`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_ncc_xx_4.3.0_3.0_1675156926295.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_ncc_xx_4.3.0_3.0_1675156926295.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_ncc","xx") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_ncc","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_ncc| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|1.6 GB| ## References - https://huggingface.co/north/t5_small_NCC - https://github.com/google-research/text-to-text-transfer-transformer - https://github.com/google-research/t5x - https://arxiv.org/abs/2104.09617 - https://arxiv.org/abs/2104.09617 - https://arxiv.org/pdf/1910.10683.pdf - https://sites.research.google/trc/about/ --- layout: model title: English asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy TFWav2Vec2ForCTC from rafiulrumy author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy` is a English model originally trained by rafiulrumy. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy_en_4.2.0_3.0_1664102298062.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy_en_4.2.0_3.0_1664102298062.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_hindi_demo_colab_by_rafiulrumy| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: BERT Sentence Embeddings trained on Wikipedia and BooksCorpus and fine-tuned on QNLI author: John Snow Labs name: sent_bert_wiki_books_qnli date: 2021-08-31 tags: [en, open_source, sentence_embeddings, wikipedia_dataset, books_corpus_dataset, qnli_dataset] task: Embeddings language: en nav_key: models edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses a BERT base architecture initialized from https://tfhub.dev/google/experts/bert/wiki_books/1 and fine-tuned on QNLI. This is a BERT base architecture but some changes have been made to the original training and export scheme based on more recent learnings. This model is intended to be used for a variety of English NLP tasks. The pre-training data contains more formal text and the model may not generalize to more colloquial text such as social media or messages. This model is fine-tuned on the QNLI and is recommended for use in question-based natural language inference tasks. The QNLI fine-tuning task where is a classification task for a question, context pair, whether the context contains the answer and where the context paragraphs are drawn from Wikipedia. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_qnli_en_3.2.0_3.0_1630412734083.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_bert_wiki_books_qnli_en_3.2.0_3.0_1630412734083.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_qnli", "en") \ .setInputCols("sentence") \ .setOutputCol("bert_sentence") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, sent_embeddings ]) ``` ```scala val sent_embeddings = BertSentenceEmbeddings.pretrained("sent_bert_wiki_books_qnli", "en") .setInputCols("sentence") .setOutputCol("bert_sentence") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, sent_embeddings )) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] sent_embeddings_df = nlu.load('en.embed_sentence.bert.wiki_books_qnli').predict(text, output_level='sentence') sent_embeddings_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_bert_wiki_books_qnli| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[bert_sentence]| |Language:|en| |Case sensitive:|false| ## Data Source [1]: [Wikipedia dataset](https://dumps.wikimedia.org/) [2]: [BooksCorpus dataset](http://yknzhu.wixsite.com/mbweb) [3]: [QNLI dataset](https://gluebenchmark.com/) This Model has been imported from: https://tfhub.dev/google/experts/bert/wiki_books/qnli/2 --- layout: model title: Zero-shot Relation Extraction (BioBert) author: John Snow Labs name: re_zeroshot_biobert date: 2022-04-05 tags: [zero, shot, zero_shot, en, licensed] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.5.0 spark_version: 3.0 supported: true annotator: ZeroShotRelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Zero-shot Relation Extraction to extract relations between clinical entities with no training dataset, just pretrained BioBert embeddings (included in the model). This model requires Healthcare NLP 3.5.0. Take a look at how it works in the "Open in Colab" section below. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.3.ZeroShot_Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_zeroshot_biobert_en_3.5.0_3.0_1649176740466.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_zeroshot_biobert_en_3.5.0_3.0_1649176740466.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} {% raw %} ```python documenter = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = nlp.Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("tokens") sentencer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") words_embedder = nlp.WordEmbeddingsModel() \ .pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"]) \ .setOutputCol("embeddings") ner_clinical = medical.NerModel() \ .pretrained("ner_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens", "embeddings"]) \ .setOutputCol("ner_clinical") ner_clinical_converter = nlp.NerConverter() \ .setInputCols(["sentences", "tokens", "ner_clinical"]) \ .setOutputCol("ner_clinical_chunks")\ .setWhiteList(["PROBLEM", "TEST"]) # PROBLEM-TEST-TREATMENT ner_posology = medical.NerModel.pretrained("ner_posology", "en", "clinical/models") \ .setInputCols(["sentences", "tokens", "embeddings"]) \ .setOutputCol("ner_posology") ner_posology_converter = nlp.NerConverter() \ .setInputCols(["sentences", "tokens", "ner_posology"]) \ .setOutputCol("ner_posology_chunks")\ .setWhiteList(["DRUG"]) # DRUG-FREQUENCY-DOSAGE-DURATION-FORM-ROUTE-STRENGTH chunk_merger = medical.ChunkMergeApproach()\ .setInputCols("ner_clinical_chunks", "ner_posology_chunks")\ .setOutputCol('merged_ner_chunks') ## ZERO-SHOT RE Starting... re_model = medical.ZeroShotRelationExtractionModel.pretrained("re_zeroshot_biobert", "en", "clinical/models")\ .setInputCols(["merged_ner_chunks", "sentences"]) \ .setOutputCol("relations")\ .setMultiLabel(True) re_model.setRelationalCategories({ "ADE": ["{DRUG} causes {PROBLEM}."], "IMPROVE": ["{DRUG} improves {PROBLEM}.", "{DRUG} cures {PROBLEM}."], "REVEAL": ["{TEST} reveals {PROBLEM}."]}) pipeline = nlp.Pipeline() \ .setStages([documenter, tokenizer, sentencer, words_embedder, ner_clinical, ner_clinical_converter, ner_posology, ner_posology_converter, chunk_merger, re_model]) data = spark.createDataFrame( [["Paracetamol can alleviate headache or sickness. An MRI test can be used to find cancer."]] ).toDF("text") results = pipeline.fit(data).transform(data) results\ .selectExpr("explode(relations) as relation")\ .show(truncate=False) ``` ```scala val data = spark.createDataFrame(Seq("Paracetamol can alleviate headache or sickness. An MRI test can be used to find cancer.")).toDF("text") val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("tokens") val sentencer = new SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val ner_clinical = MedicalNerModel() .pretrained("ner_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_clinical") val ner_clinical_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_clinical")) .setOutputCol("ner_clinical_chunks") .setWhiteList(Array("PROBLEM", "TEST")) # PROBLEM-TEST-TREATMENT val ner_posology = MedicalNerModel() .pretrained("ner_posology", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_posology") val ner_posology_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_posology")) .setOutputCol("ner_posology_chunks") .setWhiteList(Array("DRUG")) # DRUG-FREQUENCY-DOSAGE-DURATION-FORM-ROUTE-STRENGTH val chunk_merger = ChunkMergeApproach() .setInputCols("ner_clinical_chunks", "ner_posology_chunks") .setOutputCol('merged_ner_chunks') ## ZERO-SHOT RE Starting... val re_model = ZeroShotRelationExtractionModel.pretrained("re_zeroshot_biobert", "en", "clinical/models") .setInputCols(Array("ner_chunks", "sentences")) .setOutputCol("relations") .setMultiLabel(true) re_model.setRelationalCategories({ Map( "CURE" -> Array("{TREATMENT} cures {PROBLEM}."), "IMPROVE" -> Array("{TREATMENT} improves {PROBLEM}.", "{TREATMENT} cures {PROBLEM}."), "REVEAL" -> Array("{TEST} reveals {PROBLEM}.") )) val pipeline = new Pipeline() .setStages(Array(documenter, tokenizer, sentencer, words_embedder, ner_clinical, ner_clinical_converter, ner_posology, ner_posology_converter, chunk_merger, re_model)) val model = pipeline.fit(data) val results = model.transform(data) ``` {% endraw %} {:.nlu-block} ```python import nlu nlu.load("en.relation.zeroshot_biobert").predict("""Paracetamol can alleviate headache or sickness. An MRI test can be used to find cancer.""") ```

## Results ```bash +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |relation | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |{category, 268, 358, IMPROVE, {entity1_begin -> 0, relation -> IMPROVE, hypothesis -> Paracetamol improves sickness., confidence -> 0.98819494, nli_prediction -> entail, entity1 -> DRUG, syntactic_distance -> undefined, chunk2 -> sickness, entity2_end -> 45, entity1_end -> 10, entity2_begin -> 38, entity2 -> PROBLEM, chunk1 -> Paracetamol, sentence -> 0}, []}| |{category, 0, 90, IMPROVE, {entity1_begin -> 0, relation -> IMPROVE, hypothesis -> Paracetamol improves headache., confidence -> 0.9929625, nli_prediction -> entail, entity1 -> DRUG, syntactic_distance -> undefined, chunk2 -> headache, entity2_end -> 33, entity1_end -> 10, entity2_begin -> 26, entity2 -> PROBLEM, chunk1 -> Paracetamol, sentence -> 0}, []} | |{category, 536, 615, REVEAL, {entity1_begin -> 48, relation -> REVEAL, hypothesis -> An MRI test reveals cancer., confidence -> 0.9760039, nli_prediction -> entail, entity1 -> TEST, syntactic_distance -> undefined, chunk2 -> cancer, entity2_end -> 85, entity1_end -> 58, entity2_begin -> 80, entity2 -> PROBLEM, chunk1 -> An MRI test, sentence -> 1}, []} | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_zeroshot_biobert| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References As it is a zero-shot relation extractor, no training dataset is necessary. --- layout: model title: BERT Embeddings (Large Cased) author: John Snow Labs name: bert_large_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a deep bidirectional transformer trained on Wikipedia and the BookCorpus. The details are described in the paper "[BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_large_cased_en_2.6.0_2.4_1598340717429.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_large_cased_en_2.6.0_2.4_1598340717429.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("bert_large_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("bert_large_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.large_cased').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_large_cased_embeddings I [-0.5893247723579407, -1.1389378309249878, -0.... love [-0.8002289533615112, -0.15043185651302338, 0.... NLP [-0.8995863199234009, 0.08327484875917435, 0.9... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_large_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/bert_cased_L-24_H-1024_A-16/1](https://tfhub.dev/google/bert_cased_L-24_H-1024_A-16/1) --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from FabianWillner) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_trivia_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-triviaqa-finetuned-squad` is a English model originally trained by `FabianWillner`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_trivia_squad_en_4.3.0_3.0_1672773863211.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_trivia_squad_en_4.3.0_3.0_1672773863211.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_trivia_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_trivia_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_trivia_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/FabianWillner/distilbert-base-uncased-finetuned-triviaqa-finetuned-squad --- layout: model title: Chinese T5ForConditionalGeneration Cased model (from hululuzhu) author: John Snow Labs name: t5_chinese_poem_mengzi_finetune date: 2023-01-30 tags: [zh, open_source, t5, tensorflow] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-poem-t5-mengzi-finetune` is a Chinese model originally trained by `hululuzhu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_chinese_poem_mengzi_finetune_zh_4.3.0_3.0_1675100525010.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_chinese_poem_mengzi_finetune_zh_4.3.0_3.0_1675100525010.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_chinese_poem_mengzi_finetune","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_chinese_poem_mengzi_finetune","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_chinese_poem_mengzi_finetune| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|1.0 GB| ## References - https://huggingface.co/hululuzhu/chinese-poem-t5-mengzi-finetune - https://github.com/hululuzhu/chinese-ai-writing-share - https://github.com/hululuzhu/chinese-ai-writing-share/tree/main/slides - https://github.com/chinese-poetry/chinese-poetry --- layout: model title: Spanish RobertaForQuestionAnswering (from mrm8488) author: John Snow Labs name: roberta_qa_RuPERTa_base_finetuned_squadv2 date: 2022-06-20 tags: [es, open_source, question_answering, roberta] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RuPERTa-base-finetuned-squadv2` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_RuPERTa_base_finetuned_squadv2_es_4.0.0_3.0_1655727371159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_RuPERTa_base_finetuned_squadv2_es_4.0.0_3.0_1655727371159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_RuPERTa_base_finetuned_squadv2","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_RuPERTa_base_finetuned_squadv2","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.squadv2.roberta.base_v2").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_RuPERTa_base_finetuned_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|es| |Size:|470.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-squadv2 --- layout: model title: Extract Mentions of Response to Cancer Treatment author: John Snow Labs name: ner_oncology_response_to_treatment date: 2022-11-24 tags: [licensed, clinical, en, oncology, ner, treatment] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts entities related to the patient”s response to the oncology treatment, including clinical response and changes in tumor size. Definitions of Predicted Entities: - `Line_Of_Therapy`: Explicit references to the line of therapy of an oncological therapy (e.g. "first-line treatment"). - `Response_To_Treatment`: Terms related to clinical progress of the patient related to cancer treatment, including "recurrence", "bad response" or "improvement". - `Size_Trend`: Terms related to the changes in the size of the tumor (such as "growth" or "reduced in size"). ## Predicted Entities `Line_Of_Therapy`, `Response_To_Treatment`, `Size_Trend` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_en_4.2.2_3.0_1669307329775.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_response_to_treatment_en_4.2.2_3.0_1669307329775.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------|:----------------------| | first-line | Line_Of_Therapy | | recurrence | Response_To_Treatment | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_response_to_treatment| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|34.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Response_To_Treatment 326 101 157 483 0.76 0.67 0.72 Size_Trend 43 28 70 113 0.61 0.38 0.47 Line_Of_Therapy 99 11 7 106 0.90 0.93 0.92 macro_avg 468 140 234 702 0.76 0.66 0.70 micro_avg 468 140 234 702 0.76 0.67 0.71 ``` --- layout: model title: Legal Restricted Stock Unit Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_restricted_stock_unit_agreement date: 2022-12-06 tags: [en, legal, classification, agreement, restricted, stock, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_restricted_stock_unit_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `restricted-stock-unit-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `restricted-stock-unit-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_restricted_stock_unit_agreement_en_1.0.0_3.0_1670358322057.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_restricted_stock_unit_agreement_en_1.0.0_3.0_1670358322057.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[restricted-stock-unit-agreement]| |[other]| |[other]| |[restricted-stock-unit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_restricted_stock_unit_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.99 0.99 0.99 80 restricted-stock-unit-agreement 0.97 0.97 0.97 39 accuracy - - 0.98 119 macro-avg 0.98 0.98 0.98 119 weighted-avg 0.98 0.98 0.98 119 ``` --- layout: model title: Greek BertForQuestionAnswering Cased model (from Danastos) author: John Snow Labs name: bert_qa_triviaqa_el_4 date: 2022-07-07 tags: [el, open_source, bert, question_answering] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `triviaqa_bert_el_4` is a Greek model originally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_triviaqa_el_4_el_4.0.0_3.0_1657193016800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_triviaqa_el_4_el_4.0.0_3.0_1657193016800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_triviaqa_el_4","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_triviaqa_el_4","el") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_triviaqa_el_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|el| |Size:|421.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/triviaqa_bert_el_4 --- layout: model title: Spanish Electra Uncased Embeddings (Oscar dataset) author: John Snow Labs name: electra_embeddings_electricidad_base_generator date: 2022-05-17 tags: [es, open_source, electra, embeddings] task: Embeddings language: es edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electricidad-base-generator` is a Spanish model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electricidad_base_generator_es_3.4.4_3.0_1652786783374.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electricidad_base_generator_es_3.4.4_3.0_1652786783374.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electricidad_base_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|126.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/mrm8488/electricidad-base-generator - https://imgur.com/uxAvBfh - https://oscar-corpus.com/ - https://openreview.net/pdf?id=r1xMH1BtvB - https://arxiv.org/pdf/1406.2661.pdf - https://rajpurkar.github.io/SQuAD-explorer/ - https://openreview.net/pdf?id=r1xMH1BtvB - https://twitter.com/julien_c - https://twitter.com/mrm8488 --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from transformersbook) author: John Snow Labs name: xlmroberta_ner_transformersbook_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `transformersbook`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_transformersbook_base_finetuned_panx_all_xx_4.1.0_3.0_1660429047165.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_transformersbook_base_finetuned_panx_all_xx_4.1.0_3.0_1660429047165.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_transformersbook_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_transformersbook_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_transformersbook_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/transformersbook/xlm-roberta-base-finetuned-panx-all - https://learning.oreilly.com/library/view/natural-language-processing/9781098103231/ - https://github.com/nlp-with-transformers/notebooks/blob/main/04_multilingual-ner.ipynb - https://paperswithcode.com/sota?task=Token+Classification&dataset=wikiann --- layout: model title: Text Cleaning author: John Snow Labs name: text_cleaning date: 2022-07-07 tags: [en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The text_cleaning is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and cleans text. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/text_cleaning_en_4.0.0_3.0_1657198002660.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/text_cleaning_en_4.0.0_3.0_1657198002660.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("text_cleaning", "en") result = pipeline.annotate("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|text_cleaning| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|934.6 KB| ## Included Models - DocumentAssembler - TokenizerModel - NormalizerModel - StopWordsCleaner - LemmatizerModel - TokenAssembler --- layout: model title: BioBERT Sentence Embeddings (Pubmed Large) author: John Snow Labs name: sent_biobert_pubmed_large_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_pubmed_large_cased_en_2.6.2_2.4_1600531709085.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_pubmed_large_cased_en_2.6.2_2.4_1600531709085.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pubmed_large_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pubmed_large_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.biobert.pubmed_large_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_biobert_pubmed_large_cased_embeddings sentence [-0.50444495677948, 0.15660151839256287, -0.03... I hate cancer [0.3133466839790344, 0.1945181041955948, 0.014... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_pubmed_large_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.2| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|1024| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: Fast Neural Machine Translation Model from English to Pedi author: John Snow Labs name: opus_mt_en_nso date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, nso, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `nso` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_nso_xx_2.7.0_2.4_1609165051207.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_nso_xx_2.7.0_2.4_1609165051207.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_nso| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from krinal214) author: John Snow Labs name: bert_qa_bert_all_squad_all_translated date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-all-squad_all_translated` is a English model orginally trained by `krinal214`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_all_squad_all_translated_en_4.0.0_3.0_1654179485871.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_all_squad_all_translated_en_4.0.0_3.0_1654179485871.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_all_squad_all_translated","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_all_squad_all_translated","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_translated.bert.by_krinal214").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_all_squad_all_translated| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/krinal214/bert-all-squad_all_translated --- layout: model title: Clinical QA BioGPT (JSL - conversational) author: John Snow Labs name: biogpt_chat_jsl_conversational date: 2023-04-18 tags: [clinical, licensed, en, tensorflow] task: Text Generation language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalTextGenerator article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is based on BioGPT finetuned with medical conversations happening in a clinical settings and can answer clinical questions related to symptoms, drugs, tests, and diseases. The difference between this model and `biogpt_chat_jsl` is that this model produces more concise/smaller response. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/biogpt_chat_jsl_conversational_en_4.4.0_3.0_1681853305199.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/biogpt_chat_jsl_conversational_en_4.4.0_3.0_1681853305199.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash question: How to treat asthma ? answer: You have to take montelukast + albuterol tablet once or twice in day according to severity of symptoms. Montelukast is used as a maintenance therapy to relieve symptoms of asthma. Albuterol is used as a rescue therapy when symptoms are severe. You can also use inhaled corticosteroids ( ICS ) like budesonide or fluticasone for long term treatment. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biogpt_chat_jsl_conversational| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.4 GB| |Case sensitive:|true| --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_cline_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cline_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_squad2.0_en_4.0.0_3.0_1655728028363.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cline_squad2.0_en_4.0.0_3.0_1655728028363.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_cline_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_cline_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.cline.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cline_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/cline_squad2.0 --- layout: model title: English asr_wav2vec2_vee_demo_colab TFWav2Vec2ForCTC from KISSz author: John Snow Labs name: asr_wav2vec2_vee_demo_colab date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_vee_demo_colab` is a English model originally trained by KISSz. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_vee_demo_colab_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_vee_demo_colab_en_4.2.0_3.0_1664104883104.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_vee_demo_colab_en_4.2.0_3.0_1664104883104.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_vee_demo_colab", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_vee_demo_colab", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_vee_demo_colab| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|3.2 MB| --- layout: model title: Detect Persons, Locations and Organization Entities in Turkish (GloVe 840B_300) author: John Snow Labs name: turkish_ner_840B_300 date: 2020-11-10 task: Named Entity Recognition language: tr edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [tr, open_source] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Pretrained Named Entity Recognition (NER) deep learning model for Turkish texts. It recognizes Persons, Locations, and Organization entities using multi-lingual Bert word embedding. The SparkNLP deep learning model (NerDL) is inspired by a former state of the art model for NER ç Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. {:.h2_title} ## Predicted Entities Persons-``PER``, Locations-``LOC``, Organizations-``ORG``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_TR/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_TR.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/turkish_ner_840B_300_tr_2.6.2_2.4_1605042988496.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/turkish_ner_840B_300_tr_2.6.2_2.4_1605042988496.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use Use as part of an NLP pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... ner_model = NerDLModel.pretrained("turkish_ner_840B_300", "tr") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (28 Ekim 1955 doğumlu), Amerikalı bir iş adamı, yazılım geliştirici, yatırımcı ve hayırseverdir. En çok Microsoft şirketinin kurucu ortağı olarak bilinir. William Gates , Microsoft şirketindeki kariyeri boyunca başkan, icra kurulu başkanı, başkan ve yazılım mimarisi başkanı pozisyonlarında bulunmuş, aynı zamanda Mayıs 2014'e kadar en büyük bireysel hissedar olmuştur. O, 1970'lerin ve 1980'lerin mikrobilgisayar devriminin en tanınmış girişimcilerinden ve öncülerinden biridir. Seattle Washington'da doğup büyüyen William Gates, 1975'te New Mexico Albuquerque'de çocukluk arkadaşı Paul Allen ile Microsoft şirketini kurdu; dünyanın en büyük kişisel bilgisayar yazılım şirketi haline geldi. William Gates, Ocak 2000'de icra kurulu başkanı olarak istifa edene kadar şirketi başkan ve icra kurulu başkanı olarak yönetti ve daha sonra yazılım mimarisi başkanı oldu. 1990'ların sonlarında, William Gates rekabete aykırı olduğu düşünülen iş taktikleri nedeniyle eleştirilmişti. Bu görüş, çok sayıda mahkeme kararıyla onaylanmıştır. Haziran 2006'da William Gates, Microsoft şirketinde yarı zamanlı bir göreve ve 2000 yılında eşi Melinda Gates ile birlikte kurdukları özel hayır kurumu olan B&Melinda G. Vakfı'nda tam zamanlı çalışmaya geçeceğini duyurdu. Görevlerini kademeli olarak Ray Ozzie ve Craig Mundie'ye devretti. Şubat 2014'te Microsoft başkanlığından ayrıldı ve yeni atanan icra kurulu başkanı, Satya Nadella'yı desteklemek için teknoloji danışmanı olarak yeni bir göreve başladı."]], ["text"])) ``` ```scala ... val ner_model = NerDLModel.pretrained("turkish_ner_840B_300", "tr") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("William Henry Gates III (28 Ekim 1955 doğumlu), Amerikalı bir iş adamı, yazılım geliştirici, yatırımcı ve hayırseverdir. En çok Microsoft şirketinin kurucu ortağı olarak bilinir. William Gates , Microsoft şirketindeki kariyeri boyunca başkan, icra kurulu başkanı, başkan ve yazılım mimarisi başkanı pozisyonlarında bulunmuş, aynı zamanda Mayıs 2014"e kadar en büyük bireysel hissedar olmuştur. O, 1970"lerin ve 1980"lerin mikrobilgisayar devriminin en tanınmış girişimcilerinden ve öncülerinden biridir. Seattle Washington"da doğup büyüyen William Gates, 1975"te New Mexico Albuquerque"de çocukluk arkadaşı Paul Allen ile Microsoft şirketini kurdu; dünyanın en büyük kişisel bilgisayar yazılım şirketi haline geldi. William Gates, Ocak 2000"de icra kurulu başkanı olarak istifa edene kadar şirketi başkan ve icra kurulu başkanı olarak yönetti ve daha sonra yazılım mimarisi başkanı oldu. 1990"ların sonlarında, William Gates rekabete aykırı olduğu düşünülen iş taktikleri nedeniyle eleştirilmişti. Bu görüş, çok sayıda mahkeme kararıyla onaylanmıştır. Haziran 2006"da William Gates, Microsoft şirketinde yarı zamanlı bir göreve ve 2000 yılında eşi Melinda Gates ile birlikte kurdukları özel hayır kurumu olan B&Melinda G. Vakfı"nda tam zamanlı çalışmaya geçeceğini duyurdu. Görevlerini kademeli olarak Ray Ozzie ve Craig Mundie"ye devretti. Şubat 2014"te Microsoft başkanlığından ayrıldı ve yeni atanan icra kurulu başkanı, Satya Nadella'yı desteklemek için teknoloji danışmanı olarak yeni bir göreve başladı.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (28 Ekim 1955 doğumlu), Amerikalı bir iş adamı, yazılım geliştirici, yatırımcı ve hayırseverdir. En çok Microsoft şirketinin kurucu ortağı olarak bilinir. William Gates , Microsoft şirketindeki kariyeri boyunca başkan, icra kurulu başkanı, başkan ve yazılım mimarisi başkanı pozisyonlarında bulunmuş, aynı zamanda Mayıs 2014'e kadar en büyük bireysel hissedar olmuştur. O, 1970'lerin ve 1980'lerin mikrobilgisayar devriminin en tanınmış girişimcilerinden ve öncülerinden biridir. Seattle Washington'da doğup büyüyen William Gates, 1975'te New Mexico Albuquerque'de çocukluk arkadaşı Paul Allen ile Microsoft şirketini kurdu; dünyanın en büyük kişisel bilgisayar yazılım şirketi haline geldi. William Gates, Ocak 2000'de icra kurulu başkanı olarak istifa edene kadar şirketi başkan ve icra kurulu başkanı olarak yönetti ve daha sonra yazılım mimarisi başkanı oldu. 1990'ların sonlarında, William Gates rekabete aykırı olduğu düşünülen iş taktikleri nedeniyle eleştirilmişti. Bu görüş, çok sayıda mahkeme kararıyla onaylanmıştır. Haziran 2006'da William Gates, Microsoft şirketinde yarı zamanlı bir göreve ve 2000 yılında eşi Melinda Gates ile birlikte kurdukları özel hayır kurumu olan B&Melinda G. Vakfı'nda tam zamanlı çalışmaya geçeceğini duyurdu. Görevlerini kademeli olarak Ray Ozzie ve Craig Mundie'ye devretti. Şubat 2014'te Microsoft başkanlığından ayrıldı ve yeni atanan icra kurulu başkanı, Satya Nadella'yı desteklemek için teknoloji danışmanı olarak yeni bir göreve başladı."""] ner_df = nlu.load('tr.ner').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +-------------------------+---------+ |chunk |ner_label| +-------------------------+---------+ |William Henry Gates III |PER | |Microsoft |ORG | |William Gates |PER | |Microsoft |ORG | |Seattle Washington'da |LOC | |William Gates |PER | |New Mexico Albuquerque'de|LOC | |Paul Allen |PER | |Microsoft |ORG | |William Gates |PER | |William Gates |PER | |William Gates |PER | |Microsoft |ORG | |Melinda Gates |PER | |B&Melinda G. Vakfı'nda |ORG | |Ray Ozzie |PER | |Craig Mundie'ye |PER | |Microsoft |ORG | |Satya Nadella'yı |PER | +-------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|turkish_ner_840B_300| |Type:|ner| |Compatibility:|Spark NLP 2.6.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|tr| |Dependencies:|glove_840B_300| {:.h2_title} ## Data Source Trained on a custom dataset with multi-lingual GloVe Embeddings ``glove_840B_300``. {:.h2_title} ## Benchmarking ```bash label tp fp fn prec rec f1 B-LOC 2081 157 142 0.9298481 0.93612236 0.9329747 I-ORG 1292 220 152 0.8544974 0.8947368 0.87415427 I-LOC 293 81 66 0.78342247 0.81615597 0.79945433 I-PER 1578 127 99 0.9255132 0.940966 0.9331757 B-ORG 1846 185 145 0.9089119 0.9271723 0.91795135 B-PER 3043 186 206 0.942397 0.93659586 0.9394875 tp: 10133 fp: 956 fn: 810 labels: 6 Macro-average prec: 0.890765, rec: 0.9086249, f1: 0.8996063 Micro-average prec: 0.91378844, rec: 0.9259801, f1: 0.9198439 ``` --- layout: model title: Legal Construction Clause Binary Classifier author: John Snow Labs name: legclf_construction_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `construction` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `construction` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_construction_clause_en_1.0.0_3.2_1660123351617.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_construction_clause_en_1.0.0_3.2_1660123351617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_construction_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[construction]| |[other]| |[other]| |[construction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_construction_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support construction 0.88 0.84 0.86 43 other 0.90 0.93 0.92 70 accuracy - - 0.89 113 macro-avg 0.89 0.88 0.89 113 weighted-avg 0.89 0.89 0.89 113 ``` --- layout: model title: English BertForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: bert_qa_bert_large_uncased_whole_word_masking_chaii date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-whole-word-masking-chaii` is a English model orginally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_chaii_en_4.0.0_3.0_1654536887751.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_uncased_whole_word_masking_chaii_en_4.0.0_3.0_1654536887751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_uncased_whole_word_masking_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_uncased_whole_word_masking_chaii","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.bert.large_uncased_uncased_whole_word_masking.by_SauravMaheshkar").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_uncased_whole_word_masking_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/bert-large-uncased-whole-word-masking-chaii --- layout: model title: Explain Document Pipeline - CRA author: John Snow Labs name: explain_clinical_doc_cra class: PipelineModel language: en nav_key: models repository: clinical/models date: 2020-08-19 task: [Named Entity Recognition, Assertion Status, Relation Extraction, Pipeline Healthcare] edition: Healthcare NLP 2.5.5 spark_version: 2.4 tags: [clinical,licensed,pipeline,en] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description A pretrained pipeline with `ner_clinical`, `assertion_dl`, `re_clinical`. It will extract clinical entities, assign assertion status and find relationships between clinical entities. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.Pretrained_Clinical_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_cra_en_2.5.5_2.4_1597846145640.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/explain_clinical_doc_cra_en_2.5.5_2.4_1597846145640.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPython.html %} ```python cra_pipeline = PretrainedPipeline("explain_clinical_doc_cra","en","clinical/models") annotations = cra_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache.She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)[0] annotations.keys() ``` ```scala val cra_pipeline = new PretrainedPipeline("explain_clinical_doc_cra","en","clinical/models") val result = cra_pipeline.fullAnnotate("""She is admitted to The John Hopkins Hospital 2 days ago with a history of gestational diabetes mellitus diagnosed. She denied pain and any headache.She was seen by the endocrinology service and she was discharged on 03/02/2018 on 40 units of insulin glargine, 12 units of insulin lispro, and metformin 1000 mg two times a day. She had close follow-up with endocrinology post discharge. """)(0) ```

{:.h2_title} ## Results This pretrained pipeline gives the result of `ner_clinical`, `re_clinical` and `assertion_dl` models. ```bash | | chunk1 | ner_clinical1 | assertion | relation | chunk2 | ner_clinical2 | |---|-------------------------------|---------------|------------|----------|---------------|---------------| | 0 | gestational diabetes mellitus | PROBLEM | present | TrAP | metformin | TREATMENT | | 1 | gestational diabetes mellitus | PROBLEM | present | TrAP | polyuria | PROBLEM | | 2 | gestational diabetes mellitus | PROBLEM | present | TrCP | polydipsia | PROBLEM | | 3 | gestational diabetes mellitus | PROBLEM | present | TrCP | poor appetite | PROBLEM | | 4 | metformin | TREATMENT | present | TrAP | polyuria | PROBLEM | | 5 | metformin | TREATMENT | present | TrAP | polydipsia | PROBLEM | | 6 | metformin | TREATMENT | present | TrAP | poor appetite | PROBLEM | | 7 | polydipsia | PROBLEM | present | TrAP | vomiting | PROBLEM | ``` {:.model-param} ## Model Information {:.table-model} |---------------|--------------------------| | Name: | explain_clinical_doc_cra | | Type: | PipelineModel | | Compatibility: | Spark NLP 2.5.5+ | | License: | Licensed | | Edition: | Official | | Language: | en | {:.h2_title} ## Included Models - ner_clinical - assertion_dl - re_clinical --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-32-finetuned-squad-seed-10` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10_en_4.3.0_3.0_1674215255291.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10_en_4.3.0_3.0_1674215255291.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_32_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|417.6 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-32-finetuned-squad-seed-10 --- layout: model title: English BertForQuestionAnswering model (from phiyodr) author: John Snow Labs name: bert_qa_bert_base_finetuned_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-finetuned-squad2` is a English model orginally trained by `phiyodr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_finetuned_squad2_en_4.0.0_3.0_1654179945636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_finetuned_squad2_en_4.0.0_3.0_1654179945636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_finetuned_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_finetuned_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/phiyodr/bert-base-finetuned-squad2 - https://arxiv.org/abs/1810.04805 - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json - https://rajpurkar.github.io/SQuAD-explorer/ - https://arxiv.org/abs/1806.03822 --- layout: model title: Extract relations between effects of using multiple drugs (ReDL) author: John Snow Labs name: redl_drug_drug_interaction_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract potential improvements or harmful effects of Drug-Drug interactions (DDIs) when two or more drugs are taken at the same time or at certain interval ## Predicted Entities `DDI-advise`, `DDI-effect`, `DDI-false`, `DDI-int`, `DDI-mechanism` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_DRUG_DRUG_INT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_2.7.3_2.4_1612441748775.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_2.7.3_2.4_1612441748775.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_dpr_spanish_question_encoder_allqa_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|412.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/IIC/dpr-spanish-question_encoder-allqa-base - https://arxiv.org/abs/2004.04906 - https://github.com/facebookresearch/DPR - https://arxiv.org/abs/2004.04906 - https://paperswithcode.com/sota?task=text+similarity&dataset=squad_es --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_53_greek_by_lighteternal TFWav2Vec2ForCTC from lighteternal author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_greek_by_lighteternal` is a Modern Greek (1453-) model originally trained by lighteternal. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal_el_4.2.0_3.0_1664105186590.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal_el_4.2.0_3.0_1664105186590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal', lang = 'el') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal", lang = "el") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_greek_by_lighteternal| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|el| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English XlmRoBertaForQuestionAnswering (from vesteinn) author: John Snow Labs name: xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `XLMr-ENIS-QA-IsQ-EnA` is a English model originally trained by `vesteinn`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA_en_4.0.0_3.0_1655984020629.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA_en_4.0.0_3.0_1655984020629.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.xlmr_roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_XLMr_ENIS_QA_IsQ_EnA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|457.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/vesteinn/XLMr-ENIS-QA-IsQ-EnA --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-32-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1655732633509.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1655732633509.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_32d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_32_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|417.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-32-finetuned-squad-seed-4 --- layout: model title: Part of Speech for Portuguese author: John Snow Labs name: pos_ud_bosque date: 2021-03-08 tags: [part_of_speech, open_source, portuguese, pos_ud_bosque, pt] task: Part of Speech Tagging language: pt edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - DET - NOUN - ADJ - PROPN - AUX - ADP - PUNCT - NUM - ADV - VERB - PRON - CCONJ - X - SCONJ - INTJ - PART - SYM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bosque_pt_3.0.0_3.0_1615230225146.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bosque_pt_3.0.0_3.0_1615230225146.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_bosque", "pt") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Olá de John Snow Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_bosque", "pt") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Olá de John Snow Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Olá de John Snow Labs! ""] token_df = nlu.load('pt.pos').predict(text) token_df ```

## Results ```bash token pos 0 Olá PROPN 1 de ADP 2 John PROPN 3 Snow PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bosque| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|pt| --- layout: model title: Pipeline to Detect Genes/Proteins (BC2GM) in Medical Texts author: John Snow Labs name: ner_biomedical_bc2gm_pipeline date: 2023-03-14 tags: [bc2gm, ner, biomedical, gene_protein, gene, protein, en, licensed, clinical] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_biomedical_bc2gm](https://nlp.johnsnowlabs.com/2022/05/11/ner_biomedical_bc2gm_en_2_4.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_biomedical_bc2gm_pipeline_en_4.3.0_3.2_1678787724252.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_biomedical_bc2gm_pipeline_en_4.3.0_3.2_1678787724252.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_biomedical_bc2gm_pipeline", "en", "clinical/models") text = '''Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_biomedical_bc2gm_pipeline", "en", "clinical/models") val text = "Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biomedical_bc2gm.pipeline").predict("""Immunohistochemical staining was positive for S-100 in all 9 cases stained, positive for HMB-45 in 9 (90%) of 10, and negative for cytokeratin in all 9 cases in which myxoid melanoma remained in the block after previous sections.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------|--------:|------:|:-------------|-------------:| | 0 | S-100 | 46 | 50 | GENE_PROTEIN | 0.9911 | | 1 | HMB-45 | 89 | 94 | GENE_PROTEIN | 0.9944 | | 2 | cytokeratin | 131 | 141 | GENE_PROTEIN | 0.9951 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_biomedical_bc2gm_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Italian BertForMaskedLM Base Uncased model (from dbmdz) author: John Snow Labs name: bert_embeddings_base_italian_uncased date: 2022-12-02 tags: [it, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: it edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-italian-uncased` is a Italian model originally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_uncased_it_4.2.4_3.0_1670017957577.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_italian_uncased_it_4.2.4_3.0_1670017957577.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_uncased","it") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_italian_uncased","it") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_italian_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|412.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-uncased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Multilingual XLMRoBerta Embeddings (from hfl) author: John Snow Labs name: xlmroberta_embeddings_cino_base_v2 date: 2022-05-13 tags: [zh, ko, open_source, xlm_roberta, embeddings, xx, cino] task: Embeddings language: xx edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: XlmRoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `cino-base-v2` is a Multilingual model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_base_v2_xx_3.4.4_3.0_1652439334973.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_embeddings_cino_base_v2_xx_3.4.4_3.0_1652439334973.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_cino_base_v2| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|xx| |Size:|712.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/cino-base-v2 - https://github.com/ymcui/Chinese-Minority-PLM - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology --- layout: model title: Notice Clause NER Model author: John Snow Labs name: legner_notice_clause date: 2022-12-16 tags: [en, legal, ner, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model aimed to be used in notice clauses, to retrieve entities as NOTICE_METHOD, NOTICE_PARTY, ADDRESS, EMAIL, etc. Make sure you run this model only on notice clauses, after you filter them using `legclf_notice_clause` ## Predicted Entities `ADDRESS`, `DEPARTMENT`, `EMAIL`, `FAX`, `NAME`, `NOTICE_METHOD`, `NOTICE_PARTY`, `PERSON`, `PHONE`, `TITLE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_notice_clause_en_1.0.0_3.0_1671211179919.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_notice_clause_en_1.0.0_3.0_1671211179919.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------------------------------------------------------+------------+ |ner_chunk |label | +---------------------------------------------------------------------+------------+ |ExxonMobil |NOTICE_PARTY| |ExxonMobil Research and Engineering Company |NAME | |1545 Route 22 East Annandale, NJ 08801-0900 |ADDRESS | |Timothy Barckholtz |PERSON | |Senior Scientific Advisor |TITLE | |tim.barckholtz@exxonmobil.com |EMAIL | +---------------------------------------------------------------------+------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_notice_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|1.1 MB| ## References In-house dataset ## Benchmarking ```bash label precision recall f1-score support ADDRESS 0.86 0.94 0.90 141 DEPARTMENT 0.75 0.27 0.40 11 EMAIL 0.92 1.00 0.96 48 FAX 0.65 0.88 0.75 51 NAME 0.78 0.79 0.79 140 NOTICE_METHOD 0.74 0.80 0.77 353 NOTICE_PARTY 0.77 0.85 0.81 103 PERSON 0.91 0.94 0.92 114 PHONE 0.60 0.47 0.53 19 TITLE 0.76 0.90 0.82 80 micro-avg 0.78 0.85 0.81 1060 macro-avg 0.77 0.79 0.77 1060 weighted-avg 0.79 0.85 0.81 1060 ``` --- layout: model title: Legal Organization Clause Binary Classifier author: John Snow Labs name: legclf_organization_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `organization` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `organization` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_organization_clause_en_1.0.0_3.2_1660123788612.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_organization_clause_en_1.0.0_3.2_1660123788612.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_organization_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[organization]| |[other]| |[other]| |[organization]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_organization_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support organization 0.99 0.91 0.95 100 other 0.96 1.00 0.98 207 accuracy - - 0.97 307 macro-avg 0.97 0.95 0.96 307 weighted-avg 0.97 0.97 0.97 307 ``` --- layout: model title: Chinese Bert Embeddings (4-layer) author: John Snow Labs name: bert_embeddings_rbt4 date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `rbt4` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_zh_3.4.2_3.0_1649669853254.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_rbt4_zh_3.4.2_3.0_1649669853254.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_rbt4| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|171.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/rbt4 - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: English asr_wav2vec2_base_test TFWav2Vec2ForCTC from cahya author: John Snow Labs name: asr_wav2vec2_base_test date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_test` is a English model originally trained by cahya. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_test_en_4.2.0_3.0_1664035642510.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_test_en_4.2.0_3.0_1664035642510.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_test", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_test", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_test| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|348.7 MB| --- layout: model title: Dispute Clauses NER author: John Snow Labs name: legner_dispute_clauses date: 2023-01-18 tags: [en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Legal NER model which helps to retrieve Courts/Arbitrations, Rules and Resolution Means from legal agreements. ## Predicted Entities `RESOLUT_MEANS`, `RULES_NAME`, `COURT_NAME` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_dispute_clauses_en_1.0.0_3.0_1674054944954.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_dispute_clauses_en_1.0.0_3.0_1674054944954.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------+---------------+ | token| ner_label| +-------------+---------------+ | The| O| | contract| O| | includes| O| | a| O| | dispute| O| | clause| O| | that| O| | requires| O| | the| O| | parties| O| | to| O| | follow| O| | the| O| | rules| O| | of| O| | judicial|B-RESOLUT_MEANS| | arbitration|I-RESOLUT_MEANS| | set| O| | forth| O| | by| O| | the| O| | United| B-RULES_NAME| | Nations| I-RULES_NAME| | Commission| I-RULES_NAME| | on| I-RULES_NAME| |International| I-RULES_NAME| | Trade| I-RULES_NAME| | Law| I-RULES_NAME| | (| I-RULES_NAME| | UNCITRAL| I-RULES_NAME| | )| I-RULES_NAME| | Rules| I-RULES_NAME| | of| I-RULES_NAME| | Arbitration| I-RULES_NAME| | and| O| | the| O| | jurisdiction| O| | of| O| | the| O| |International| B-COURT_NAME| | Chamber| I-COURT_NAME| | of| I-COURT_NAME| | Commerce| I-COURT_NAME| | court| O| | in| O| | the| O| | event| O| | of| O| | a| O| | dispute| O| | .| O| +-------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_dispute_clauses| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References In-house annotations of the CUAD dataset ## Benchmarking ```bash label tp fp fn prec rec f1 B-RESOLUT_MEANS 14 4 6 0.7777778 0.7 0.73684216 B-RULES_NAME 15 0 5 1.0 0.75 0.85714287 I-RESOLUT_MEANS 12 0 3 1.0 0.8 0.88888896 B-COURT_NAME 26 6 6 0.8125 0.8125 0.8125 I-RULES_NAME 101 7 19 0.9351852 0.84166664 0.8859649 I-COURT_NAME 166 23 24 0.87830687 0.8736842 0.87598944 Macro-average 334 40 63 0.9006283 0.7963085 0.8452619 Micro-average 334 40 63 0.8930481 0.84130985 0.8664072 ``` --- layout: model title: Chinese Bert Embeddings (Base, Wobert model) author: John Snow Labs name: bert_embeddings_wobert_chinese_base date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `wobert_chinese_base` is a Chinese model orginally trained by `junnyu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_wobert_chinese_base_zh_3.4.2_3.0_1649669889842.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_wobert_chinese_base_zh_3.4.2_3.0_1649669889842.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_wobert_chinese_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|419.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/junnyu/wobert_chinese_base - https://github.com/ZhuiyiTechnology/WoBERT - https://github.com/JunnYu/WoBERT_pytorch --- layout: model title: Turkish Named Entity Recognition (from winvoker) author: John Snow Labs name: bert_ner_bert_base_turkish_cased_ner_tf date: 2022-05-09 tags: [bert, ner, token_classification, tr, open_source] task: Named Entity Recognition language: tr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-turkish-cased-ner-tf` is a Turkish model orginally trained by `winvoker`. ## Predicted Entities `LOC`, `PER`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_turkish_cased_ner_tf_tr_3.4.2_3.0_1652099181398.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_base_turkish_cased_ner_tf_tr_3.4.2_3.0_1652099181398.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_base_turkish_cased_ner_tf| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|tr| |Size:|412.9 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/winvoker/bert-base-turkish-cased-ner-tf - https://github.com/stefan-it/turkish-bert/files/4558187/nerdata.txt - https://ieeexplore.ieee.org/document/7495744 --- layout: model title: English image_classifier_vit_rust_image_classification_6 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_6 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_6` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust`, `rust` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_6_en_4.1.0_3.0_1660168572229.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_6_en_4.1.0_3.0_1660168572229.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_6", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_6", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_6| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering Large Cased model (from LucasS) author: John Snow Labs name: bert_qa_bertlargeabsa date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bertLargeABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bertlargeabsa_en_4.0.0_3.0_1657188817667.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bertlargeabsa_en_4.0.0_3.0_1657188817667.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bertlargeabsa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_bertlargeabsa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bertlargeabsa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LucasS/bertLargeABSA --- layout: model title: Multilingual BertForQuestionAnswering Cased model (from roshnir) author: John Snow Labs name: bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev date: 2022-07-07 tags: [xx, open_source, bert, question_answering] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `mBert-finetuned-mlqa-dev-en-zh-hi` is a Multilingual model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev_xx_4.0.0_3.0_1657190064489.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev_xx_4.0.0_3.0_1657190064489.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE"]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev","xx") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("PUT YOUR QUESTION HERE", "PUT YOUR CONTEXT HERE").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_mbert_finetuned_mlqa_en_zh_hi_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/mBert-finetuned-mlqa-dev-en-zh-hi --- layout: model title: Relation Extraction between anatomical entities and other clinical entities author: John Snow Labs name: re_oncology_location_wip date: 2022-09-27 tags: [licensed, clinical, oncology, en, relation_extraction, anatomy] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links extractions from anatomical entities (such as Site_Breast or Site_Lung) to other clinical entities (such as Tumor_Finding or Cancer_Surgery). ## Predicted Entities `is_location_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_oncology_location_wip_en_4.0.0_3.0_1664301554732.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_oncology_location_wip_en_4.0.0_3.0_1664301554732.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use relation pairs to include only the combinations of entities that are relevant in your case.

## Results ```bash +--------------+-------------+-------------+-----------+------+---------------+-------------+-----------+------+----------+ | relation| entity1|entity1_begin|entity1_end|chunk1| entity2|entity2_begin|entity2_end|chunk2|confidence| +--------------+-------------+-------------+-----------+------+---------------+-------------+-----------+------+----------+ |is_location_of|Tumor_Finding| 35| 38| lump|Anatomical_Site| 53| 58|breast|0.81353307| +--------------+-------------+-------------+-----------+------+---------------+-------------+-----------+------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_oncology_location_wip| |Type:|re| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Size:|266.7 KB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.83 0.93 0.88 is_location_of 0.93 0.84 0.88 macro-avg 0.88 0.88 0.88 ``` --- layout: model title: Recognize Entities OntoNotes - ELECTRA Small author: John Snow Labs name: onto_recognize_entities_electra_small date: 2020-12-09 task: [Named Entity Recognition, Sentence Detection, Embeddings, Pipeline Public] language: en nav_key: models edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [en, open_source, pipeline] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A pre-trained pipeline containing NerDl Model. The NER model trained on OntoNotes 5.0 with `electra_small_uncased` embeddings. It can extract up to following 18 entities: ## Predicted Entities `CARDINAL`, `DATE`, `EVENT`, `FAC`, `GPE`, `LANGUAGE`, `LAW`, `LOC`, `MONEY`, `NORP`, `ORDINAL`, `ORG`, `PERCENT`, `PERSON`, `PRODUCT`, `QUANTITY`, `TIME`, `WORK_OF_ART`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_small_en_2.7.0_2.4_1607511710029.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_small_en_2.7.0_2.4_1607511710029.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('onto_recognize_entities_electra_small') result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("onto_recognize_entities_electra_small") val result = pipeline.annotate("Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament.") ``` {:.nlu-block} ```python import nlu text = ["""Johnson first entered politics when elected in 2001 as a member of Parliament. He then served eight years as the mayor of London, from 2008 to 2016, before rejoining Parliament."""] ner_df = nlu.load('en.ner.onto.electra.small').predict(text, output_level='chunk') ner_df[["entities", "entities_class"]] ```

{:.h2_title} ## Results ```bash +------------+---------+ |chunk |ner_label| +------------+---------+ |Johnson |PERSON | |first |ORDINAL | |2001 |DATE | |eight years |DATE | |London |GPE | |2008 to 2016|DATE | +------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_electra_small| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|en| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - Tokenizer - BertEmbeddings - NerDLModel - NerConverter --- layout: model title: German asr_exp_w2v2t_r_wav2vec2_s466 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2t_r_wav2vec2_s466 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_r_wav2vec2_s466` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2t_r_wav2vec2_s466_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_r_wav2vec2_s466_de_4.2.0_3.0_1664108739065.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_r_wav2vec2_s466_de_4.2.0_3.0_1664108739065.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2t_r_wav2vec2_s466", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2t_r_wav2vec2_s466", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2t_r_wav2vec2_s466| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: English BertForMaskedLM Base Uncased model (from model-attribution-challenge) author: John Snow Labs name: bert_embeddings_model_attribution_challenge_base_uncased date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased` is a English model originally trained by `model-attribution-challenge`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_model_attribution_challenge_base_uncased_en_4.2.4_3.0_1670019231783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_model_attribution_challenge_base_uncased_en_4.2.4_3.0_1670019231783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_model_attribution_challenge_base_uncased","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_model_attribution_challenge_base_uncased","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_model_attribution_challenge_base_uncased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|409.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/model-attribution-challenge/bert-base-uncased - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://github.com/google-research/bert/blob/master/README.md - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: Dutch RoBERTa Embeddings (from CLTL) author: John Snow Labs name: roberta_embeddings_MedRoBERTa.nl date: 2022-04-14 tags: [roberta, embeddings, nl, open_source] task: Embeddings language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `MedRoBERTa.nl` is a Dutch model orginally trained by `CLTL`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_MedRoBERTa.nl_nl_3.4.2_3.0_1649949062891.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_MedRoBERTa.nl_nl_3.4.2_3.0_1649949062891.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_MedRoBERTa.nl| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|nl| |Size:|472.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/CLTL/MedRoBERTa.nl - https://github.com/cltl-students/verkijk_stella_rma_thesis_dutch_medical_language_model --- layout: model title: Turkish BertForTokenClassification Cased model (from alierenak) author: John Snow Labs name: bert_token_classifier_berturk_cased_ner date: 2022-11-30 tags: [tr, open_source, bert, token_classification, ner, tensorflow] task: Named Entity Recognition language: tr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `berturk-cased-ner` is a Turkish model originally trained by `alierenak`. ## Predicted Entities `LOCATION`, `ORGANIZATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_berturk_cased_ner_tr_4.2.4_3.0_1669815532606.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_berturk_cased_ner_tr_4.2.4_3.0_1669815532606.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_berturk_cased_ner","tr") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = BertForTokenClassification.pretrained("bert_token_classifier_berturk_cased_ner","tr") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_berturk_cased_ner| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|tr| |Size:|412.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/alierenak/berturk-cased-ner - https://www.cambridge.org/core/journals/natural-language-engineering/article/abs/statistical-information-extraction-system-for-turkish/7C288FAFC71D5F0763C1F8CE66464017 - https://aclanthology.org/P11-3019 - https://data.tdd.ai/#/effafb5f-ebfc-4e5c-9a63-4f709ec1a135 --- layout: model title: ALBERT XLarge CoNNL-03 NER Pipeline author: ahmedlone127 name: albert_xlarge_token_classifier_conll03_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, albert, conll03, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [albert_xlarge_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/26/albert_xlarge_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/albert_xlarge_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655219194773.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/albert_xlarge_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655219194773.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("albert_xlarge_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.") ``` ```scala val pipeline = new PretrainedPipeline("albert_xlarge_token_classifier_conll03_pipeline", lang = "en") pipeline.annotate("My name is John and I work at John Snow Labs.")) ```

## Results ```bash +---------------------------------------------------------------+------+ |text |result| +---------------------------------------------------------------+------+ |The patient took Advil and he experienced an adverse reaction. |[Male]| +---------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_gender_biobert| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.0 MB| |Case sensitive:|true| |Max sentence length:|128| ## References This model is trained on more than four thousands clinical documents (radiology reports, pathology reports, clinical visits, etc) annotated internally. ## Benchmarking ```bash label precision recall f1-score support Female 0.94 0.94 0.94 479 Male 0.88 0.86 0.87 245 Unknown 0.73 0.78 0.76 102 accuracy 0.89 0.89 0.89 826 macro-avg 0.85 0.86 0.85 826 weighted-avg 0.90 0.89 0.90 826 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from pszemraj) author: John Snow Labs name: t5_base_askscience date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-askscience` is a English model originally trained by `pszemraj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_base_askscience_en_4.3.0_3.0_1675108079371.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_base_askscience_en_4.3.0_3.0_1675108079371.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_base_askscience","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_base_askscience","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_askscience| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|1.0 GB| ## References - https://huggingface.co/pszemraj/t5-base-askscience --- layout: model title: English BertForQuestionAnswering Large Uncased model (from tli8hf) author: John Snow Labs name: bert_qa_unqover_large_uncased_newsqa date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unqover-bert-large-uncased-newsqa` is a English model originally trained by `tli8hf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_unqover_large_uncased_newsqa_en_4.0.0_3.0_1657193094308.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_unqover_large_uncased_newsqa_en_4.0.0_3.0_1657193094308.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_unqover_large_uncased_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_unqover_large_uncased_newsqa","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_unqover_large_uncased_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/tli8hf/unqover-bert-large-uncased-newsqa --- layout: model title: Stopwords Remover for Sinhalese language (191 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, si, open_source] task: Stop Words Removal language: si edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_si_3.4.1_3.0_1646673160202.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_si_3.4.1_3.0_1646673160202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------+ |result | +------------------+ |[ඔබ, මට, හොඳ, නැත]| +------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|si| |Size:|2.2 KB| --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739733256.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739733256.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_twostagequadruplet_hier_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|306.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_twostagequadruplet_hier_epochs_1_shard_1_squad2.0 --- layout: model title: Japanese DistilBERT Embeddings (from Geotrend) author: John Snow Labs name: distilbert_embeddings_distilbert_base_ja_cased date: 2022-04-12 tags: [distilbert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `distilbert-base-ja-cased` is a Japanese model orginally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_ja_cased_ja_3.4.2_3.0_1649783582188.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_distilbert_base_ja_cased_ja_3.4.2_3.0_1649783582188.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_distilbert_base_ja_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|188.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/distilbert-base-ja-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: English BertForMaskedLM Cased model (from bioformers) author: John Snow Labs name: bert_embeddings_bioformer_cased_v1.0 date: 2022-12-02 tags: [en, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bioformer-cased-v1.0` is a English model originally trained by `bioformers`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bioformer_cased_v1.0_en_4.2.4_3.0_1670020877701.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bioformer_cased_v1.0_en_4.2.4_3.0_1670020877701.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_bioformer_cased_v1.0","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_bioformer_cased_v1.0","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bioformer_cased_v1.0| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|159.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/bioformers/bioformer-cased-v1.0 - https://allenai.github.io/scispacy/ - https://biocreative.bioinformatics.udel.edu/media/store/files/2021/TRACK5_pos_1_BC7_submission_221.pdf - https://github.com/WGLab/bioformer/issues --- layout: model title: Spanish Part of Speech Tagger (from mrm8488) author: John Snow Labs name: bert_pos_bert_spanish_cased_finetuned_pos_syntax date: 2022-05-09 tags: [bert, pos, part_of_speech, es, open_source] task: Part of Speech Tagging language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-spanish-cased-finetuned-pos-syntax` is a Spanish model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_spanish_cased_finetuned_pos_syntax_es_3.4.2_3.0_1652091508354.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_spanish_cased_finetuned_pos_syntax_es_3.4.2_3.0_1652091508354.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_spanish_cased_finetuned_pos_syntax| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|410.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mrm8488/bert-spanish-cased-finetuned-pos-syntax - https://github.com/dccuchile/beto - https://www.kaggle.com/nltkdata/conll-corpora - https://www.kaggle.com/nltkdata/conll-corpora - https://twitter.com/mrm8488 --- layout: model title: Legal Ownership Clause Binary Classifier author: John Snow Labs name: legclf_ownership_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `ownership` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `ownership` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_ownership_clause_en_1.0.0_3.2_1660123796143.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_ownership_clause_en_1.0.0_3.2_1660123796143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_ownership_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[ownership]| |[other]| |[other]| |[ownership]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_ownership_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.95 0.95 59 ownership 0.91 0.91 0.91 32 accuracy - - 0.93 91 macro-avg 0.93 0.93 0.93 91 weighted-avg 0.93 0.93 0.93 91 ``` --- layout: model title: Legal Remedies Clause Binary Classifier author: John Snow Labs name: legclf_remedies_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `remedies` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `remedies` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_remedies_clause_en_1.0.0_3.2_1660123931699.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_remedies_clause_en_1.0.0_3.2_1660123931699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_remedies_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[remedies]| |[other]| |[other]| |[remedies]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_remedies_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.95 0.96 0.96 56 remedies 0.92 0.88 0.90 26 accuracy - - 0.94 82 macro-avg 0.93 0.92 0.93 82 weighted-avg 0.94 0.94 0.94 82 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from eAsyle) author: John Snow Labs name: roberta_qa_base_custom date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta_base_custom_QA` is a English model originally trained by `eAsyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_custom_en_4.3.0_3.0_1674223065625.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_custom_en_4.3.0_3.0_1674223065625.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_custom","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_custom","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_custom| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|424.1 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/eAsyle/roberta_base_custom_QA --- layout: model title: Detect Living Species(embeddings_scielo_300d) author: John Snow Labs name: ner_living_species_300 date: 2022-07-26 tags: [es, ner, licensed, clinical] task: Named Entity Recognition language: es edition: Healthcare NLP 3.5.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Spanish which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `embeddings_scielo_300d` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_es_3.5.0_3.0_1658876470162.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_300_es_3.5.0_3.0_1658876470162.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_300", "es","clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("embeddings_scielo_300d","es","clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_300", "es","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("es.med_ner.living_species.300").predict("""Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_cpener_test","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_r_ab_test_by_pablouribe', lang = 'ab') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_r_ab_test_by_pablouribe", lang = "ab") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_r_ab_test_by_pablouribe| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ab| |Size:|451.4 KB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English asr_wav2vec2_base_960h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_base_960h date: 2022-09-23 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_960h` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_960h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_en_4.2.0_3.0_1663934856069.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_en_4.2.0_3.0_1663934856069.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_960h", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_960h", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_960h| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.6 MB| --- layout: model title: Finnish asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot TFWav2Vec2ForCTC from aapot author: John Snow Labs name: asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot` is a Finnish model originally trained by aapot. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot_fi_4.2.0_3.0_1664024535110.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot_fi_4.2.0_3.0_1664024535110.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xlsr_300m_finnish_lm_by_aapot| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: BioBERT Embeddings (PMC) author: John Snow Labs name: biobert_pmc_base_cased date: 2020-09-19 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.2 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_pmc_base_cased_en_2.6.2_2.4_1600530421096.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_pmc_base_cased_en_2.6.2_2.4_1600530421096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash token en_embed_biobert_pmc_base_cased_embeddings I [0.0654267892241478, 0.06330983340740204, 0.13... hate [0.3058323264122009, 0.4778319299221039, -0.09... cancer [0.3130614757537842, 0.024675076827406883, -0.... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.2| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: XLNet Large CoNLL-03 NER Pipeline author: ahmedlone127 name: xlnet_large_token_classifier_conll03_pipeline date: 2022-06-14 tags: [open_source, ner, token_classifier, xlnet, conll03, large, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlnet_large_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/09/28/xlnet_large_token_classifier_conll03_en.html) model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/xlnet_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655218011796.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/xlnet_large_token_classifier_conll03_pipeline_en_4.0.0_3.0_1655218011796.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_cline_emanuals_techqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_cline_emanuals_techqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.roberta.techqa_cline_emanuals.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cline_emanuals_techqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/cline-emanuals-techqa --- layout: model title: Fast Neural Machine Translation Model from English to Turkic Languages author: John Snow Labs name: opus_mt_en_trk date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, trk, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `trk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_trk_xx_2.7.0_2.4_1609167167159.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_trk_xx_2.7.0_2.4_1609167167159.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_trk| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from Raynok) author: John Snow Labs name: roberta_qa_squad date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-squad` is a English model originally trained by `Raynok`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_squad_en_4.3.0_3.0_1674222368310.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_squad_en_4.3.0_3.0_1674222368310.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Raynok/roberta-squad --- layout: model title: Smaller BERT Embeddings (L-12_H-128_A-2) author: John Snow Labs name: small_bert_L12_128 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L12_128_en_2.6.0_2.4_1598344378220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L12_128_en_2.6.0_2.4_1598344378220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L12_128", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L12_128", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L12_128').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L12_128_embeddings I [-0.19200828671455383, -1.1298311948776245, -0... love [-0.427543580532074, -1.2282991409301758, -0.2... NLP [0.5775153636932373, 0.06353635340929031, -0.3... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L12_128| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|128| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab7_by_sameearif88 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab7_by_sameearif88` is a English model originally trained by sameearif88. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88_en_4.2.0_3.0_1664020412505.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88_en_4.2.0_3.0_1664020412505.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab7_by_sameearif88| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline to Detect clinical entities (ner_jsl_enriched_biobert) author: John Snow Labs name: ner_jsl_enriched_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_enriched_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_jsl_enriched_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_biobert_pipeline_en_4.3.0_3.2_1679316183988.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_enriched_biobert_pipeline_en_4.3.0_3.2_1679316183988.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_jsl_enriched_biobert_pipeline", "en", "clinical/models") text = '''The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_jsl_enriched_biobert_pipeline", "en", "clinical/models") val text = "The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_enriched_biobert.pipeline").predict("""The patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:----------------------------|--------:|------:|:-------------|-------------:| | 0 | 21-day-old | 17 | 26 | Age | 1 | | 1 | male | 38 | 41 | Gender | 0.9326 | | 2 | mom | 75 | 77 | Gender | 0.9258 | | 3 | she | 147 | 149 | Gender | 0.8551 | | 4 | mild | 168 | 171 | Modifier | 0.8119 | | 5 | problems with his breathing | 173 | 199 | Symptom_Name | 0.624975 | | 6 | negative | 220 | 227 | Negation | 0.9946 | | 7 | perioral cyanosis | 237 | 253 | Symptom_Name | 0.41775 | | 8 | retractions | 258 | 268 | Symptom_Name | 0.9572 | | 9 | mom | 285 | 287 | Gender | 0.9468 | | 10 | Tylenol | 345 | 351 | Drug_Name | 0.989 | | 11 | His | 400 | 402 | Gender | 0.8694 | | 12 | his | 488 | 490 | Gender | 0.8967 | | 13 | respiratory congestion | 492 | 513 | Symptom_Name | 0.4195 | | 14 | He | 516 | 517 | Gender | 0.8529 | | 15 | tired | 550 | 554 | Symptom_Name | 0.7902 | | 16 | fussy | 569 | 573 | Symptom_Name | 0.9389 | | 17 | albuterol | 637 | 645 | Drug_Name | 0.9588 | | 18 | His | 675 | 677 | Gender | 0.8484 | | 19 | he | 721 | 722 | Gender | 0.8909 | | 20 | he | 778 | 779 | Gender | 0.8625 | | 21 | Mom | 821 | 823 | Gender | 0.8167 | | 22 | denies | 825 | 830 | Negation | 0.9841 | | 23 | diarrhea | 836 | 843 | Symptom_Name | 0.6033 | | 24 | His | 846 | 848 | Gender | 0.8459 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_enriched_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.3 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from English to Chinyanja author: John Snow Labs name: opus_mt_en_ny date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, ny, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `ny` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_ny_xx_2.7.0_2.4_1609169030913.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_ny_xx_2.7.0_2.4_1609169030913.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_ny| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Civil Law Document Classifier (EURLEX) author: John Snow Labs name: legclf_civil_law_bert date: 2023-03-06 tags: [en, legal, classification, clauses, civil_law, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_civil_law_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Civil_Law or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Civil_Law`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_civil_law_bert_en_1.0.0_3.0_1678111748956.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_civil_law_bert_en_1.0.0_3.0_1678111748956.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_civil_law_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Civil_Law]| |[Other]| |[Other]| |[Civil_Law]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_civil_law_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Civil_Law 0.89 0.89 0.89 53 Other 0.87 0.87 0.87 45 accuracy - - 0.88 98 macro-avg 0.88 0.88 0.88 98 weighted-avg 0.88 0.88 0.88 98 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from razent) author: John Snow Labs name: t5_scifive_base_pubmed_pmc date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `SciFive-base-Pubmed_PMC` is a English model originally trained by `razent`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_scifive_base_pubmed_pmc_en_4.3.0_3.0_1675099109583.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_scifive_base_pubmed_pmc_en_4.3.0_3.0_1675099109583.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_scifive_base_pubmed_pmc","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_scifive_base_pubmed_pmc","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_scifive_base_pubmed_pmc| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|474.3 MB| ## References - https://huggingface.co/razent/SciFive-base-Pubmed_PMC - https://arxiv.org/abs/2106.03598 - https://github.com/justinphan3110/SciFive --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding ICD10-CM Codes author: John Snow Labs name: snomed_icd10cm_mapping date: 2023-06-13 tags: [en, licensed, clinical, pipeline, chunk_mapping, snomed, icd10cm] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_icd10cm_mapper` model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapping_en_4.4.4_3.2_1686663521616.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_icd10cm_mapping_en_4.4.4_3.2_1686663521616.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("snomed_icd10cm_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(128041000119107 292278006 293072005) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("snomed_icd10cm_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(128041000119107 292278006 293072005) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.snomed_to_icd10cm.pipe").predict("""Put your text here.""") ```

## Results ```bash Results | | snomed_code | icd10cm_code | |---:|:----------------------------------------|:---------------------------| | 0 | 128041000119107 | 292278006 | 293072005 | K22.70 | T43.595 | T37.1X5 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_icd10cm_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.5 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC5CD_Chem_Modified_PubMedBERT_512 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC5CD-Chem-Modified-PubMedBERT-512` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CD_Chem_Modified_PubMedBERT_512_en_4.0.0_3.0_1657109249349.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CD_Chem_Modified_PubMedBERT_512_en_4.0.0_3.0_1657109249349.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC5CD_Chem_Modified_PubMedBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC5CD-Chem-Modified-PubMedBERT-512 --- layout: model title: Translate Baltic languages to English Pipeline author: John Snow Labs name: translate_bat_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bat, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bat` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bat_en_xx_2.7.0_2.4_1609686947121.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bat_en_xx_2.7.0_2.4_1609686947121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bat_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bat_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bat.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bat_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-27 tags: [it, ner, legal, mapa, licensed] task: Named Entity Recognition language: it edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Italian` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_it_1.0.0_3.0_1682597548726.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_it_1.0.0_3.0_1682597548726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_base_it_cased", "it")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "it", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""In pendenza del giudizio relativo alla responsabilità genitoriale instaurato in Italia, la sig.ra Grigorescu, il 30 settembre 2009, ha adito la Judecătoria București ( Tribunale di primo grado di Bucarest ) chiedendo il divorzio, l’affidamento esclusivo del figlio e un contributo al mantenimento del figlio a carico del padre a titolo di mantenimento della prole."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-----------------+---------+ |chunk |ner_label| +-----------------+---------+ |Italia |ADDRESS | |sig.ra Grigorescu|PERSON | |30 settembre 2009|DATE | |Bucarest |ADDRESS | +-----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|it| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 1.00 1.00 1.00 14 AMOUNT 1.00 1.00 1.00 3 DATE 1.00 1.00 1.00 45 ORGANISATION 0.89 0.89 0.89 9 PERSON 0.92 1.00 0.96 12 macro-avg 0.98 0.99 0.98 83 macro-avg 0.96 0.98 0.97 83 weighted-avg 0.98 0.99 0.98 83 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from sunitha) author: John Snow Labs name: distilbert_qa_base_uncased_3feb_2022_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-3feb-2022-finetuned-squad` is a English model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_3feb_2022_finetuned_squad_en_4.3.0_3.0_1672767357306.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_3feb_2022_finetuned_squad_en_4.3.0_3.0_1672767357306.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_3feb_2022_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_3feb_2022_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_3feb_2022_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/sunitha/distilbert-base-uncased-3feb-2022-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from Arabic to Polish author: John Snow Labs name: opus_mt_ar_pl date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, pl, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: pl {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_pl_xx_3.1.0_2.4_1622560089386.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_pl_xx_3.1.0_2.4_1622560089386.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_pl| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Roles NER author: John Snow Labs name: legner_roles date: 2023-01-02 tags: [role, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This NER models extracts legal roles in an agreement, such as `Borrower`, `Supplier`, `Agent`, `Attorney`, `Pursuant`, etc. ## Predicted Entities `ROLE`, `O` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_roles_en_1.0.0_3.0_1672673551040.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_roles_en_1.0.0_3.0_1672673551040.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencizer = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = legal.NerModel.pretrained('legner_roles', 'en', 'legal/models')\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[documenter, sentencizer, tokenizer, embeddings, ner, ner_converter]) empty = spark.createDataFrame([[example]]).toDF("text") tr_results = model.transform(spark.createDataFrame([[example]]).toDF('text')) ```

## Results ```bash +-------+---------+---------+ |sent_id|chunk |ner_label| +-------+---------+---------+ |1 |Lender |ROLE | |1 |Lender's |ROLE | |1 |principal|ROLE | |1 |Lender |ROLE | |2 |pursuant |ROLE | |3 |Lenders |ROLE | |3 |Lenders |ROLE | |3 |Lenders |ROLE | |4 |Lenders |ROLE | |7 |Agent |ROLE | |14 |Lenders |ROLE | |14 |Borrowers|ROLE | |14 |Lender |ROLE | |14 |Agent |ROLE | |14 |Lender |ROLE | |15 |Agent |ROLE | |15 |Lender |ROLE | |15 |pursuant |ROLE | |15 |Agent |ROLE | |15 |Borrowers|ROLE | +-------+---------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_roles| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References CUAD dataset and synthetic data ## Benchmarking ```bash label tp fp fn prec rec f1 B-ROLE 19095 16 77 0.9991628 0.9959837 0.9975707 I-ROLE 162 1 0 0.993865 1.0 0.9969231 Macro-average 19257 17 77 0.9965139 0.99799186 0.9972523 Micro-average 19257 17 77 0.999118 0.9960174 0.9975653 ``` --- layout: model title: English asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2 TFWav2Vec2ForCTC from gary109 author: John Snow Labs name: asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2` is a English model originally trained by gary109. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2_en_4.2.0_3.0_1664101378845.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2_en_4.2.0_3.0_1664101378845.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_ai_light_dance_singing2_wav2vec2_large_xlsr_53_5gram_v4_2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Detect Drug Information (Small) author: John Snow Labs name: ner_posology_small date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for posology, this NER is trained with the ``embeddings_clinical`` word embeddings model, so be sure to use the same embeddings in the pipeline. ## Predicted Entities ``DOSAGE``, ``DRUG``, ``DURATION``, ``FORM``, ``FREQUENCY``, ``ROUTE``, ``STRENGTH``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_en_3.0.0_3.0_1617208436385.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_en_3.0.0_3.0_1617208436385.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_posology_small","en","clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val model = MedicalNerModel.pretrained("ner_posology_small","en","clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, model, ner_converter)) val data = Seq("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology.small").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunk |ner | +--------------+---------+ |insulin |DRUG | |Bactrim |DRUG | |for 14 days |DURATION | |Fragmin |DRUG | |5000 units |DOSAGE | |subcutaneously|ROUTE | |daily |FREQUENCY| |Xenaderm |DRUG | |topically |ROUTE | |b.i.d., |FREQUENCY| |Lantus |DRUG | |40 units |DOSAGE | |subcutaneously|ROUTE | |at bedtime |FREQUENCY| |OxyContin |DRUG | |30 mg |STRENGTH | |p.o |ROUTE | |q.12 h |FREQUENCY| |folic acid |DRUG | |1 mg |STRENGTH | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_small| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on the 2018 i2b2 dataset (no FDA) with ``embeddings_clinical``. https://www.i2b2.org/NLP/Medication ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | B-DRUG | 1408 | 62 | 99 | 0.957823 | 0.934307 | 0.945919 | | 1 | B-STRENGTH | 470 | 43 | 29 | 0.916179 | 0.941884 | 0.928854 | | 2 | I-DURATION | 123 | 22 | 8 | 0.848276 | 0.938931 | 0.891304 | | 3 | I-STRENGTH | 499 | 66 | 15 | 0.883186 | 0.970817 | 0.924931 | | 4 | I-FREQUENCY | 945 | 47 | 55 | 0.952621 | 0.945 | 0.948795 | | 5 | B-FORM | 365 | 13 | 12 | 0.965608 | 0.96817 | 0.966887 | | 6 | B-DOSAGE | 298 | 27 | 26 | 0.916923 | 0.919753 | 0.918336 | | 7 | I-DOSAGE | 348 | 29 | 22 | 0.923077 | 0.940541 | 0.931727 | | 8 | I-DRUG | 208 | 25 | 60 | 0.892704 | 0.776119 | 0.830339 | | 9 | I-ROUTE | 10 | 0 | 2 | 1 | 0.833333 | 0.909091 | | 10 | B-ROUTE | 467 | 4 | 25 | 0.991507 | 0.949187 | 0.969886 | | 11 | B-DURATION | 64 | 10 | 10 | 0.864865 | 0.864865 | 0.864865 | | 12 | B-FREQUENCY | 588 | 12 | 17 | 0.98 | 0.971901 | 0.975934 | | 13 | I-FORM | 264 | 5 | 4 | 0.981413 | 0.985075 | 0.98324 | | 14 | Macro-average | 6057 | 365 | 384 | 0.93387 | 0.924277 | 0.929049 | | 15 | Micro-average | 6057 | 365 | 384 | 0.943164 | 0.940382 | 0.941771 | ``` --- layout: model title: Translate English to Luvale Pipeline author: John Snow Labs name: translate_en_lue date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, lue, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `lue` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_lue_xx_2.7.0_2.4_1609688898091.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_lue_xx_2.7.0_2.4_1609688898091.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_lue", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_lue", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.lue').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_lue| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from ZYW) author: John Snow Labs name: distilbert_qa_squad_model date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-en-de-es-model` is a Multilingual model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_model_xx_4.3.0_3.0_1672775581986.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_squad_model_xx_4.3.0_3.0_1672775581986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_model","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_squad_model","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_squad_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-en-de-es-model --- layout: model title: Korean ElectraForQuestionAnswering model (from obokkkk) author: John Snow Labs name: electra_qa_base_v3_discriminator_finetuned_klue_v4 date: 2022-06-22 tags: [ko, open_source, electra, question_answering] task: Question Answering language: ko edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `koelectra-base-v3-discriminator-finetuned-klue-v4` is a Korean model originally trained by `obokkkk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_v3_discriminator_finetuned_klue_v4_ko_4.0.0_3.0_1655922184133.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_v3_discriminator_finetuned_klue_v4_ko_4.0.0_3.0_1655922184133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_v3_discriminator_finetuned_klue_v4","ko") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_v3_discriminator_finetuned_klue_v4","ko") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("내 이름은 무엇입니까?", "제 이름은 클라라이고 저는 버클리에 살고 있습니다.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ko.answer_question.klue.electra.base.by_obokkkk").predict("""내 이름은 무엇입니까?|||"제 이름은 클라라이고 저는 버클리에 살고 있습니다.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_v3_discriminator_finetuned_klue_v4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ko| |Size:|419.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/obokkkk/koelectra-base-v3-discriminator-finetuned-klue-v4 --- layout: model title: English asr_wav2vec2_large_xls_r_300m_CN_colab TFWav2Vec2ForCTC from li666 author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_CN_colab` is a English model originally trained by li666. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab_en_4.2.0_3.0_1664094642974.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab_en_4.2.0_3.0_1664094642974.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_CN_colab| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Financial Pipeline (ORG-PER-ROLE-DATE) author: John Snow Labs name: finpipe_org_per_role_date date: 2022-09-09 tags: [en, financial, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a pretrained pipeline to extract Companies (ORG), People (PERSON), Job titles (ROLE) and Dates combining different pretrained NER models to improve coverage. ## Predicted Entities `ORG`, `PERSON`, `ROLE`, `DATE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINPIPE_ORG_PER_DATE_ROLES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finpipe_org_per_role_date_en_1.0.0_3.2_1662716423161.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finpipe_org_per_role_date_en_1.0.0_3.2_1662716423161.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from johnsnowlabs import * deid_pipeline = PretrainedPipeline("finpipe_org_per_role_date", "en", "finance/models") deid_pipeline.annotate("John Smith works as Computer Engineer at Amazon since 2020") res = deid_pipeline.annotate("John Smith works as Computer Engineer at Amazon since 2020") for token, ner in zip(res['token'], res['ner']): print(f"{token} ({ner})") ```

## Results ```bash John (B-PERSON) Smith (I-PERSON) works (O) as (O) Computer (B-ROLE) Engineer (I-ROLE) at (O) Amazon (B-ORG) since (O) 2020 (B-DATE) ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finpipe_org_per_role_date| |Type:|pipeline| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|828.4 MB| ## References In-house annotations on legal and financial documents, Ontonotes, Conll 2003, Finsec conll, Cuad dataset, 10k filings ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - FinanceNerModel - FinanceBertForTokenClassification - NerConverter - NerConverter - ChunkMergeModel --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from Evelyn18) author: John Snow Labs name: roberta_qa_base_spanish_squades_modelo_v1 date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-spanish-squades-modelo-robertav1` is a Spanish model originally trained by `Evelyn18`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo_v1_es_4.3.0_3.0_1674218326080.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_spanish_squades_modelo_v1_es_4.3.0_3.0_1674218326080.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo_v1","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_spanish_squades_modelo_v1","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_spanish_squades_modelo_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/Evelyn18/roberta-base-spanish-squades-modelo-robertav1 --- layout: model title: Spanish Named Entity Recognition (Base, Plus, CAPITEL competition at IberLEF 2020 dataset) author: John Snow Labs name: roberta_ner_roberta_base_bne_capitel_ner_plus date: 2022-05-03 tags: [roberta, ner, open_source, es] task: Named Entity Recognition language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-base-bne-capitel-ner-plus` is a Spanish model orginally trained by `PlanTL-GOB-ES`. ## Predicted Entities `ORG`, `OTH`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_base_bne_capitel_ner_plus_es_3.4.2_3.0_1651592741277.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_base_bne_capitel_ner_plus_es_3.4.2_3.0_1651592741277.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_roberta_base_bne_capitel_ner_plus| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|458.9 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/PlanTL-GOB-ES/roberta-base-bne-capitel-ner-plus - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://sites.google.com/view/capitel2020 - https://github.com/PlanTL-GOB-ES/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Lemmatizer (Germany, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, de] task: Lemmatization language: de edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Germany Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_de_3.4.1_3.0_1646316522910.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_de_3.4.1_3.0_1646316522910.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------+ |result | +--------------------------------+ |[Du, sein, nicht, gut, als, ich]| +--------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|de| |Size:|4.2 MB| --- layout: model title: Translate Bislama to English Pipeline author: John Snow Labs name: translate_bi_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, bi, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `bi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_bi_en_xx_2.7.0_2.4_1609689875128.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_bi_en_xx_2.7.0_2.4_1609689875128.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_bi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_bi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.bi.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_bi_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Extract relations between phenotypic abnormalities and diseases (ReDL) author: John Snow Labs name: redl_human_phenotype_gene_biobert date: 2023-01-14 tags: [relation_extraction, en, licensed, clinical, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract relations to fully understand the origin of some phenotypic abnormalities and their associated diseases. 1 : Entities are related, 0 : Entities are not related. ## Predicted Entities `1`, `0` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_4.2.4_3.0_1673737099610.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_human_phenotype_gene_biobert_en_4.2.4_3.0_1673737099610.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+-------------------+----------+ |relation|entity1|entity1_begin|entity1_end| chunk1|entity2|entity2_begin|entity2_end| chunk2|confidence| +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+-------------------+----------+ | 0| HP| 10| 29|retinal degeneration| HP| 32| 43| hearing loss|0.92880034| | 0| HP| 10| 29|retinal degeneration| HP| 49| 61| renal failure|0.93935645| | 0| HP| 10| 29|retinal degeneration| HP| 64| 76| short stature|0.92370766| | 1| HP| 10| 29|retinal degeneration| GENE| 96| 103| SH3PXD2B|0.63739055| | 1| HP| 10| 29|retinal degeneration| HP| 162| 180|autosomal recessive|0.58393383| | 0| HP| 32| 43| hearing loss| HP| 49| 61| renal failure| 0.9543991| | 0| HP| 32| 43| hearing loss| HP| 64| 76| short stature| 0.8060494| | 1| HP| 32| 43| hearing loss| GENE| 96| 103| SH3PXD2B| 0.8507128| | 1| HP| 32| 43| hearing loss| HP| 162| 180|autosomal recessive|0.90283227| | 0| HP| 49| 61| renal failure| HP| 64| 76| short stature|0.85388213| | 1| HP| 49| 61| renal failure| GENE| 96| 103| SH3PXD2B|0.76057386| | 1| HP| 49| 61| renal failure| HP| 162| 180|autosomal recessive|0.85482293| | 1| HP| 64| 76| short stature| GENE| 96| 103| SH3PXD2B| 0.8951201| | 1| HP| 64| 76| short stature| HP| 162| 180|autosomal recessive| 0.9018232| | 1| GENE| 96| 103| SH3PXD2B| HP| 162| 180|autosomal recessive|0.97185487| +--------+-------+-------------+-----------+--------------------+-------+-------------+-----------+-------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_human_phenotype_gene_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References Trained on a silver standard corpus of human phenotype and gene annotations and their relations. ## Benchmarking ```bash label Recall Precision F1 Support 0 0.922 0.908 0.915 129 1 0.831 0.855 0.843 71 Avg. 0.877 0.882 0.879 - ``` --- layout: model title: Abkhazian asr_xls_r_ab_test_by_baaastien TFWav2Vec2ForCTC from baaastien author: John Snow Labs name: asr_xls_r_ab_test_by_baaastien date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_baaastien` is a Abkhazian model originally trained by baaastien. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_ab_test_by_baaastien_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_baaastien_ab_4.2.0_3.0_1664020806550.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_baaastien_ab_4.2.0_3.0_1664020806550.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_ab_test_by_baaastien", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_ab_test_by_baaastien", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_ab_test_by_baaastien| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|442.4 KB| --- layout: model title: English BertForTokenClassification Cased model (from ghadeermobasher) author: John Snow Labs name: bert_ner_BC5CDR_Chem_Modified_BioBERT_384 date: 2022-07-06 tags: [bert, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BC5CDR-Chem-Modified-BioBERT-384` is a English model originally trained by `ghadeermobasher`. ## Predicted Entities `Chemical` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CDR_Chem_Modified_BioBERT_384_en_4.0.0_3.0_1657109284236.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_BC5CDR_Chem_Modified_BioBERT_384_en_4.0.0_3.0_1657109284236.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC5CDR_Chem_Modified_BioBERT_384| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|403.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC5CDR-Chem-Modified-BioBERT-384 --- layout: model title: Extract Biomarkers and Their Results author: John Snow Labs name: ner_oncology_biomarker_healthcare date: 2023-01-11 tags: [licensed, clinical, oncology, en, ner, biomarker] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts mentions of biomarkers and biomarker results from oncology texts. ## Predicted Entities `Biomarker_Result`, `Biomarker` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_ONCOLOGY_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_healthcare_en_4.2.4_3.0_1673477151495.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_biomarker_healthcare_en_4.2.4_3.0_1673477151495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------------------------------------|:-----------------| | negative | Biomarker_Result | | CK7 | Biomarker | | synaptophysin | Biomarker | | Syn | Biomarker | | chromogranin A | Biomarker | | CgA | Biomarker | | Muc5AC | Biomarker | | human epidermal growth factor receptor-2 | Biomarker | | HER2 | Biomarker | | Muc6 | Biomarker | | positive | Biomarker_Result | | CK20 | Biomarker | | Muc1 | Biomarker | | Muc2 | Biomarker | | E-cadherin | Biomarker | | p53 | Biomarker | | Ki-67 index | Biomarker | | 87% | Biomarker_Result | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_biomarker_healthcare| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|33.8 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label tp fp fn total precision recall f1 Biomarker_Result 519 78 62 581 0.87 0.89 0.88 Biomarker 828 51 98 926 0.94 0.89 0.92 macro-avg 1347 129 160 1507 0.91 0.89 0.90 micro-avg 1347 129 160 1507 0.91 0.89 0.90 ``` --- layout: model title: Sentence Embeddings - sbert mini (tuned) author: John Snow Labs name: sbert_jsl_mini_uncased date: 2021-06-30 tags: [embeddings, clinical, licensed, en] task: Embeddings language: en nav_key: models edition: Healthcare NLP 3.1.0 spark_version: 2.4 supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained to generate contextual sentence embeddings of input sentences. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_mini_uncased_en_3.1.0_2.4_1625050221194.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbert_jsl_mini_uncased_en_3.1.0_2.4_1625050221194.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sbiobert_embeddings = BertSentenceEmbeddings.pretrained("sbert_jsl_mini_uncased","en","clinical/models").setInputCols(["sentence"]).setOutputCol("sbert_embeddings") ``` ```scala val sbiobert_embeddings = BertSentenceEmbeddings .pretrained("sbert_jsl_mini_uncased","en","clinical/models") .setInputCols(Array("sentence")) .setOutputCol("sbert_embeddings") ``` {:.nlu-block} ```python import nlu nlu.load("en.embed_sentence.bert.jsl_mini_uncased").predict("""Put your text here.""") ```

## Results ```bash Gives a 768 dimensional vector representation of the sentence. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbert_jsl_mini_uncased| |Compatibility:|Healthcare NLP 3.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|false| ## Data Source Tuned on MedNLI dataset ## Benchmarking ```bash MedNLI Score Acc 0.663 STS(cos) 0.701 ``` --- layout: model title: Adverse Drug Events Classifier author: John Snow Labs name: classifierml_ade date: 2023-05-16 tags: [text_classification, ade, en, clinical, licensed] task: Text Classification language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: DocumentMLClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with the DocumentMLClassifierApproach annotator and classifies a text/sentence into two categories: True : The sentence is talking about a possible ADE False : The sentence doesn’t have any information about an ADE. The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). ## Predicted Entities `True`, `False` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierml_ade_en_4.4.1_3.0_1684249521145.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierml_ade_en_4.4.1_3.0_1684249521145.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------------------------------------------------------------------------------------+-------+ |text |result | +--------------------------------------------------------------------------------------------------------+-------+ |Toxic epidermal necrolysis resulted after 19 days of treatment with 5-fluorocytosine and amphotericin B.|[True] | |I feel great after taking tylenol |[False]| +--------------------------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierml_ade| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[prediction]| |Language:|en| |Size:|2.6 MB| ## References The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). Reference: Gurulingappa et al., Benchmark Corpus to Support Information Extraction for Adverse Drug Effects, JBI, 2012. http://www.sciencedirect.com/science/article/pii/S1532046412000615 ## Benchmarking ```bash label precision recall f1-score support False 0.90 0.94 0.92 3359 True 0.85 0.75 0.79 1364 accuracy - - 0.89 4723 macro avg 0.87 0.85 0.86 4723 weighted avg 0.89 0.89 0.89 4723 ``` --- layout: model title: Stop Words Cleaner for English author: John Snow Labs name: stopwords_en date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: en nav_key: models edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, en] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_en_en_2.5.4_2.4_1594742439135.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_en_en_2.5.4_2.4_1594742439135.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_en", "en") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Other than being the king of the north, John Snow is a an English physician and a leader in the development of anaesthesia and medical hygiene.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_en", "en") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Other than being the king of the north, John Snow is a an English physician and a leader in the development of anaesthesia and medical hygiene.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Other than being the king of the north, John Snow is a an English physician and a leader in the development of anaesthesia and medical hygiene."""] stopword_df = nlu.load('en.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=21, end=24, result='king', metadata={'sentence': '0'}), Row(annotatorType='token', begin=33, end=37, result='north', metadata={'sentence': '0'}), Row(annotatorType='token', begin=38, end=38, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=40, end=43, result='John', metadata={'sentence': '0'}), Row(annotatorType='token', begin=45, end=48, result='Snow', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_en| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|en| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: English image_classifier_vit_beer_whisky_wine_detection ViTForImageClassification from firas-spanioli author: John Snow Labs name: image_classifier_vit_beer_whisky_wine_detection date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_beer_whisky_wine_detection` is a English model originally trained by firas-spanioli. ## Predicted Entities `beer`, `whisky`, `wine` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_beer_whisky_wine_detection_en_4.1.0_3.0_1660165748295.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_beer_whisky_wine_detection_en_4.1.0_3.0_1660165748295.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_beer_whisky_wine_detection", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_beer_whisky_wine_detection", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_beer_whisky_wine_detection| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Part of Speech for Dutch author: John Snow Labs name: pos_ud_alpino date: 2020-05-04 01:47:00 +0800 task: Part of Speech Tagging language: nl edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [pos, nl] supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model annotates the part of speech of tokens in a text. The [parts of speech](https://universaldependencies.org/u/pos/) annotated include PRON (pronoun), CCONJ (coordinating conjunction), and 15 others. The part of speech model is useful for extracting the grammatical structure of a piece of text automatically. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/2da56c087da53a2fac1d51774d49939e05418e57/tutorials/Certification_Trainings/Public/6.Playground_DataFrames.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_alpino_nl_2.5.0_2.4_1588545949009.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_alpino_nl_2.5.0_2.4_1588545949009.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... pos = PerceptronModel.pretrained("pos_ud_alpino", "nl") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, pos]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Behalve dat hij de koning van het noorden is, is John Snow een Engelse arts en een leider in de ontwikkeling van anesthesie en medische hygiëne.") ``` ```scala ... val pos = PerceptronModel.pretrained("pos_ud_alpino", "nl") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, pos)) val data = Seq("Behalve dat hij de koning van het noorden is, is John Snow een Engelse arts en een leider in de ontwikkeling van anesthesie en medische hygiëne.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Behalve dat hij de koning van het noorden is, is John Snow een Engelse arts en een leider in de ontwikkeling van anesthesie en medische hygiëne."""] pos_df = nlu.load('nl.pos.ud_alpino').predict(text, output_level='token') pos_df ```

{:.h2_title} ## Results ```bash [Row(annotatorType='pos', begin=0, end=6, result='SCONJ', metadata={'word': 'Behalve'}), Row(annotatorType='pos', begin=8, end=10, result='SCONJ', metadata={'word': 'dat'}), Row(annotatorType='pos', begin=12, end=14, result='PRON', metadata={'word': 'hij'}), Row(annotatorType='pos', begin=16, end=17, result='DET', metadata={'word': 'de'}), Row(annotatorType='pos', begin=19, end=24, result='NOUN', metadata={'word': 'koning'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_alpino| |Type:|pos| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[pos]| |Language:|nl| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Indonesian BertForQuestionAnswering model (from Rifky) author: John Snow Labs name: bert_qa_Indobert_QA date: 2022-06-02 tags: [id, open_source, question_answering, bert] task: Question Answering language: id edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Indobert-QA` is a Indonesian model orginally trained by `Rifky`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_Indobert_QA_id_4.0.0_3.0_1654176733545.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_Indobert_QA_id_4.0.0_3.0_1654176733545.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_Indobert_QA","id") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_Indobert_QA","id") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("id.answer_question.indo_bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_Indobert_QA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|id| |Size:|412.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Rifky/Indobert-QA - https://indolem.github.io/ - https://github.com/rifkybujana/IndoBERT-QA - https://github.com/Wikidepia/indonesian_datasets/tree/master/question-answering/squad --- layout: model title: Chinese T5ForConditionalGeneration Small Cased model (from thu-coai) author: John Snow Labs name: t5_longlm_small date: 2023-01-30 tags: [zh, open_source, t5, tensorflow] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `LongLM-small` is a Chinese model originally trained by `thu-coai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_longlm_small_zh_4.3.0_3.0_1675098245048.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_longlm_small_zh_4.3.0_3.0_1675098245048.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_longlm_small","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_longlm_small","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_longlm_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|288.2 MB| ## References - https://huggingface.co/thu-coai/LongLM-small - https://jianguanthu.github.io/ - http://coai.cs.tsinghua.edu.cn/ --- layout: model title: Fast Neural Machine Translation Model from Galician to English author: John Snow Labs name: opus_mt_gl_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, gl, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `gl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_gl_en_xx_2.7.0_2.4_1609169484710.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_gl_en_xx_2.7.0_2.4_1609169484710.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_gl_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Farming Systems Document Classifier (EURLEX) author: John Snow Labs name: legclf_farming_systems_bert date: 2023-03-06 tags: [en, legal, classification, clauses, farming_systems, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_farming_systems_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Farming_Systems or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Farming_Systems`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_farming_systems_bert_en_1.0.0_3.0_1678111765449.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_farming_systems_bert_en_1.0.0_3.0_1678111765449.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_farming_systems_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Farming_Systems]| |[Other]| |[Other]| |[Farming_Systems]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_farming_systems_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Farming_Systems 0.82 0.81 0.82 52 Other 0.80 0.82 0.81 50 accuracy - - 0.81 102 macro-avg 0.81 0.81 0.81 102 weighted-avg 0.81 0.81 0.81 102 ``` --- layout: model title: Pipeline to Extract Cancer Therapies and Posology Information author: John Snow Labs name: ner_oncology_unspecific_posology_pipeline date: 2023-03-09 tags: [licensed, clinical, oncology, en, ner, treatment, posology] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_unspecific_posology](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_unspecific_posology_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_pipeline_en_4.3.0_3.2_1678347063020.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_unspecific_posology_pipeline_en_4.3.0_3.2_1678347063020.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_unspecific_posology_pipeline", "en", "clinical/models") text = '''The patient underwent a regimen consisting of adriamycin (60 mg/m2) and cyclophosphamide (600 mg/m2) over six courses. She is currently receiving his second cycle of chemotherapy and is in good overall condition.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_unspecific_posology_pipeline", "en", "clinical/models") val text = "The patient underwent a regimen consisting of adriamycin (60 mg/m2) and cyclophosphamide (600 mg/m2) over six courses. She is currently receiving his second cycle of chemotherapy and is in good overall condition." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:---------------------|-------------:| | 0 | adriamycin | 46 | 55 | Cancer_Therapy | 1 | | 1 | 60 mg/m2 | 58 | 65 | Posology_Information | 0.86955 | | 2 | cyclophosphamide | 72 | 87 | Cancer_Therapy | 1 | | 3 | 600 mg/m2 | 90 | 98 | Posology_Information | 0.81215 | | 4 | over six courses | 101 | 116 | Posology_Information | 0.9078 | | 5 | second cycle | 150 | 161 | Posology_Information | 0.9853 | | 6 | chemotherapy | 166 | 177 | Cancer_Therapy | 0.9998 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_unspecific_posology_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Sentence Entity Resolver for RxNorm (disposition) author: John Snow Labs name: sbiobertresolve_rxnorm_disposition date: 2021-08-12 tags: [rxnorm, licensed, en, entity_resolution] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.1.3 spark_version: 2.4 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps medication entities (like drugs/ingredients) to RxNorm codes and their dispositions using `sbiobert_base_cased_mli` Sentence Bert Embeddings. ## Predicted Entities Predicts RxNorm Codes, their normalized definition for each chunk, and dispositions if any. In the result, look for the aux_label parameter in the metadata to get dispositions divided by `|`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_RXNORM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_RXNORM.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_disposition_en_3.1.3_2.4_1628792971821.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_rxnorm_disposition_en_3.1.3_2.4_1628792971821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_rxnorm_disposition``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_posology``` as NER model. ```DRUG``` set in ```.setWhiteList()```.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxnorm_disposition", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sbert_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") pipelineModel = PipelineModel( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver ]) rxnorm_lp = LightPipeline(pipelineModel) result = rxnorm_lp.fullAnnotate("belimumab 80 mg/ml injectable solution") ``` ```scala val documentAssembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm_disposition", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val pipelineModel= new PipelineModel().setStages(Array(documentAssembler, sbert_embedder, rxnorm_resolver)) val rxnorm_lp = LightPipeline(pipelineModel) val result = rxnorm_lp.fullAnnotate("belimumab 80 mg/ml injectable solution") ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.rxnorm_disposition").predict("""belimumab 80 mg/ml injectable solution""") ```

## Results ```bash | | chunks | code | resolutions | all_codes | all_k_aux_labels | all_distances | |---:|:--------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------|:--------------------------------------------------------------------------------------------|:----------------------------------------------| | 0 |belimumab 80 mg/ml injectable solution | 1092440 | [belimumab 80 mg/ml injectable solution, belimumab 80 mg/ml injectable solution [benlysta], ifosfamide 80 mg/ml injectable solution, belimumab 80 mg/ml [benlysta], belimumab 80 mg/ml, ...]| [1092440, 1092444, 107034, 1092442, 1092438, ...] | [Immunomodulator, Immunomodulator, Alkylating agent, Immunomodulator, Immunomodulator, ...] | [0.0000, 0.0145, 0.0479, 0.0619, 0.0636, ...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_rxnorm_disposition| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[rxnorm_code]| |Language:|en| |Case sensitive:|false| --- layout: model title: Translate Kuanyama to English Pipeline author: John Snow Labs name: translate_kj_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, kj, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `kj` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_kj_en_xx_2.7.0_2.4_1609686306200.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_kj_en_xx_2.7.0_2.4_1609686306200.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_kj_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_kj_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.kj.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_kj_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Word2Vec Embeddings in Finnish (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, fi, open_source] task: Embeddings language: fi edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_fi_3.4.1_3.0_1647373893178.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_fi_3.4.1_3.0_1647373893178.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_chemicals", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_chemicals", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemicals").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemicals| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash +------+-------+------+------+-------+---------+------+------+ |entity| tp| fp| fn| total|precision|recall| f1| +------+-------+------+------+-------+---------+------+------+ | CHEM|58026.0|3022.0|3975.0|62001.0| 0.9505|0.9359|0.9431| +------+-------+------+------+-------+---------+------+------+ +------------------+ | macro| +------------------+ |0.9431364740875586| +------------------+ +------------------+ | micro| +------------------+ |0.9431364740875586| +------------------+ ``` --- layout: model title: English ElectraForQuestionAnswering model (from hankzhong) author: John Snow Labs name: electra_qa_hankzhong_small_discriminator_finetuned_squad date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-small-discriminator-finetuned-squad` is a English model originally trained by `hankzhong`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_hankzhong_small_discriminator_finetuned_squad_en_4.0.0_3.0_1655921254220.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_hankzhong_small_discriminator_finetuned_squad_en_4.0.0_3.0_1655921254220.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_hankzhong_small_discriminator_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_hankzhong_small_discriminator_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.small.by_hankzhong").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_hankzhong_small_discriminator_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|51.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/hankzhong/electra-small-discriminator-finetuned-squad --- layout: model title: Fast Neural Machine Translation Model from English to Morisyen author: John Snow Labs name: opus_mt_en_mfe date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, mfe, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `mfe` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_mfe_xx_2.7.0_2.4_1609168764734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_mfe_xx_2.7.0_2.4_1609168764734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_mfe| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering Cased model (from saraks) author: John Snow Labs name: distilbert_qa_cuad_document_name_cased_08_31_v1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `cuad-distil-document_name-cased-08-31-v1` is a English model originally trained by `saraks`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_document_name_cased_08_31_v1_en_4.3.0_3.0_1672766097687.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_cuad_document_name_cased_08_31_v1_en_4.3.0_3.0_1672766097687.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_document_name_cased_08_31_v1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_cuad_document_name_cased_08_31_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_cuad_document_name_cased_08_31_v1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/saraks/cuad-distil-document_name-cased-08-31-v1 --- layout: model title: Fast Neural Machine Translation Model from Bemba (Zambia) to English author: John Snow Labs name: opus_mt_bem_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, bem, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `bem` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_bem_en_xx_2.7.0_2.4_1609171066307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_bem_en_xx_2.7.0_2.4_1609171066307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_bem_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from akdeniz27) author: John Snow Labs name: roberta_qa_akdeniz27_roberta_base_cuad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-cuad` is a English model originally trained by `akdeniz27`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_akdeniz27_roberta_base_cuad_en_4.0.0_3.0_1655730413653.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_akdeniz27_roberta_base_cuad_en_4.0.0_3.0_1655730413653.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_akdeniz27_roberta_base_cuad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_akdeniz27_roberta_base_cuad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.cuad.roberta.base.by_akdeniz27").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_akdeniz27_roberta_base_cuad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|447.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/akdeniz27/roberta-base-cuad - https://github.com/TheAtticusProject/cuad - https://github.com/marshmellow77/cuad-demo --- layout: model title: English asr_wav2vec2_coral_300ep TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: asr_wav2vec2_coral_300ep date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_coral_300ep` is a English model originally trained by joaoalvarenga. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_coral_300ep_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_coral_300ep_en_4.2.0_3.0_1664023690286.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_coral_300ep_en_4.2.0_3.0_1664023690286.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_coral_300ep", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_coral_300ep", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_coral_300ep| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Arabic Bert Embeddings (from alger-ia) author: John Snow Labs name: bert_embeddings_dziribert date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `dziribert` is a Arabic model orginally trained by `alger-ia`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_dziribert_ar_3.4.2_3.0_1649677382652.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_dziribert_ar_3.4.2_3.0_1649677382652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_dziribert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|465.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/alger-ia/dziribert - https://arxiv.org/pdf/2109.12346.pdf - https://github.com/alger-ia/dziribert --- layout: model title: Greek BertForQuestionAnswering Cased model (from Danastos) author: John Snow Labs name: bert_qa_nq_squad_el_3 date: 2022-07-07 tags: [el, open_source, bert, question_answering] task: Question Answering language: el edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `nq_squad_bert_el_3` is a Greek model originally trained by `Danastos`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_nq_squad_el_3_el_4.0.0_3.0_1657190653972.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_nq_squad_el_3_el_4.0.0_3.0_1657190653972.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_nq_squad_el_3","el") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_nq_squad_el_3","el") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Ποιο είναι το όνομά μου?", "Το όνομά μου είναι Κλάρα και μένω στο Μπέρκλεϊ.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_nq_squad_el_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|el| |Size:|421.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Danastos/nq_squad_bert_el_3 --- layout: model title: English BertForQuestionAnswering model (from ruselkomp) author: John Snow Labs name: bert_qa_sbert_large_nlu_ru_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sbert_large_nlu_ru-finetuned-squad` is a English model orginally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sbert_large_nlu_ru_finetuned_squad_en_4.0.0_3.0_1654189351825.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sbert_large_nlu_ru_finetuned_squad_en_4.0.0_3.0_1654189351825.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sbert_large_nlu_ru_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_sbert_large_nlu_ru_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sbert_large_nlu_ru_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.6 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/sbert_large_nlu_ru-finetuned-squad --- layout: model title: English asr_wav2vec2_large_xlsr_sermon TFWav2Vec2ForCTC from sharonibejih author: John Snow Labs name: asr_wav2vec2_large_xlsr_sermon date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_sermon` is a English model originally trained by sharonibejih. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_sermon_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_sermon_en_4.2.0_3.0_1664115027943.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_sermon_en_4.2.0_3.0_1664115027943.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_sermon", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_sermon", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_sermon| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Translate English to Altaic languages Pipeline author: John Snow Labs name: translate_en_tut date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, tut, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `tut` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_tut_xx_2.7.0_2.4_1609689008206.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_tut_xx_2.7.0_2.4_1609689008206.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_tut", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_tut", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.tut').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_tut| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Vietnamese XlmRoBertaForQuestionAnswering (from ancs21) author: John Snow Labs name: xlm_roberta_qa_xlm_roberta_large_vi_qa date: 2022-06-23 tags: [vi, open_source, question_answering, xlmroberta] task: Question Answering language: vi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-large-vi-qa` is a Vietnamese model originally trained by `ancs21`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_vi_qa_vi_4.0.0_3.0_1655996230939.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_xlm_roberta_large_vi_qa_vi_4.0.0_3.0_1655996230939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_xlm_roberta_large_vi_qa","vi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_xlm_roberta_large_vi_qa","vi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("vi.answer_question.xlm_roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_xlm_roberta_large_vi_qa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|vi| |Size:|1.9 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ancs21/xlm-roberta-large-vi-qa - https://github.com/deepmind/xquad/blob/master/xquad.vi.json - https://github.com/mailong25/bert-vietnamese-question-answering/tree/master/dataset --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_ff12000 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-ff12000` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff12000_en_4.3.0_3.0_1675123452076.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_ff12000_en_4.3.0_3.0_1675123452076.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_ff12000","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_ff12000","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_ff12000| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|134.1 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-ff12000 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: English asr_wav2vec2_base_960h_by_facebook TFWav2Vec2ForCTC from facebook author: John Snow Labs name: asr_wav2vec2_base_960h_by_facebook date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_960h_by_facebook` is a English model originally trained by facebook. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_960h_by_facebook_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_by_facebook_en_4.2.0_3.0_1664035746899.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_960h_by_facebook_en_4.2.0_3.0_1664035746899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_960h_by_facebook", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_960h_by_facebook", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_960h_by_facebook| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.6 MB| --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_100000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-100000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_100000_cased_generator_de_3.4.4_3.0_1652786285688.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_100000_cased_generator_de_3.4.4_3.0_1652786285688.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_100000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-100000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: Translate English to Lozi Pipeline author: John Snow Labs name: translate_en_loz date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, loz, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `loz` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_loz_xx_2.7.0_2.4_1609690709587.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_loz_xx_2.7.0_2.4_1609690709587.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_loz", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_loz", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.loz').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_loz| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal NER for NDA (Dispute Resolution Clause) author: John Snow Labs name: legner_nda_dispute_resolution date: 2023-04-06 tags: [en, licensed, legal, ner, nda] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `DISPUTE_RESOL ` clause with a proper classifier (use legmulticlf_mnda_sections_paragraph_other for that purpose). It will extract the following entities: `COURT_NAME`, `LAW_LOCATION `, and `RESOLUT_MEANS `. ## Predicted Entities `COURT_NAME`, `LAW_LOCATION`, `RESOLUT_MEANS` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_dispute_resolution_en_1.0.0_3.0_1680821390209.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_dispute_resolution_en_1.0.0_3.0_1680821390209.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_dispute_resolution", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""In case no settlement can be reached through consultation within thirty ( 30 ) days after such dispute is raised, each party can submit such matter to China International Economic and Trade Arbitration Commission ( the "CIETAC") in accordance with its rules."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-------------------------------------------------------+-------------+ |chunk |ner_label | +-------------------------------------------------------+-------------+ |consultation |RESOLUT_MEANS| |China |LAW_LOCATION | |International Economic and Trade Arbitration Commission|COURT_NAME | +-------------------------------------------------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_dispute_resolution| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support COURT_NAME 0.86 0.89 0.88 75 LAW_LOCATION 0.79 0.85 0.82 79 RESOLUT_MEANS 0.88 0.88 0.88 58 micro-avg 0.84 0.87 0.85 212 macro-avg 0.84 0.87 0.86 212 weighted-avg 0.84 0.87 0.85 212 ``` --- layout: model title: English image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial ViTForImageClassification from AykeeSalazar author: John Snow Labs name: image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial` is a English model originally trained by AykeeSalazar. ## Predicted Entities `nonViolation`, `publicDrinking`, `publicSmoking` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial_en_4.1.0_3.0_1660172056548.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial_en_4.1.0_3.0_1660172056548.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vc_bantai__withoutAMBI_adunest_trial| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English BertForQuestionAnswering Base Uncased model (from AnonymousSub) author: John Snow Labs name: bert_qa_base_uncased_squad2.0 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_squad2.0_en_4.0.0_3.0_1657185907503.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_uncased_squad2.0_en_4.0.0_3.0_1657185907503.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_uncased_squad2.0","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_uncased_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.8 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/bert-base-uncased_squad2.0 --- layout: model title: Financial ORG, PER, ROLE, DATE NER author: John Snow Labs name: finner_org_per_role_date date: 2022-08-30 tags: [en, finance, ner, licensed] task: Named Entity Recognition language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is an NER model trained on SEC 10K documents, aimed to extract the following entities: - ORG - PERSON - ROLE - DATE ## Predicted Entities `ORG`, `PER`, `DATE`, `ROLE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/FINPIPE_ORG_PER_DATE_ROLES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finner_org_per_role_date_en_1.0.0_3.2_1661862246514.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finner_org_per_role_date_en_1.0.0_3.2_1661862246514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----+---+---------------------+------+ |begin|end| chunk|entity| +-----+---+---------------------+------+ | 0| 20|Jeffrey Preston Bezos|PERSON| | 37| 48| entrepreneur| ROLE| | 51| 57| founder| ROLE| | 63| 65| CEO| ROLE| | 70| 75| Amazon| ORG| +-----+---+---------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finner_org_per_role_date| |Type:|finance| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References SEC 10-K filings with in-house annotations ## Benchmarking ```bash label tp fp fn prec rec f1 B-PERSON 254 20 56 0.9270073 0.81935483 0.86986303 I-ORG 1161 133 231 0.8972179 0.8340517 0.8644826 B-DATE 202 15 14 0.9308756 0.9351852 0.9330255 I-DATE 302 29 12 0.9123867 0.96178347 0.93643415 B-ROLE 219 21 47 0.9125 0.8233083 0.8656126 B-ORG 674 92 163 0.87989557 0.80525684 0.84092325 I-ROLE 260 26 68 0.90909094 0.79268295 0.8469055 I-PERSON 501 34 94 0.9364486 0.8420168 0.88672566 Macro-average 3573 370 685 0.91317785 0.851705 0.8813709 Micro-average 3573 370 685 0.9061628 0.83912635 0.8713572 ``` --- layout: model title: Relation Extraction between different oncological entity types (ReDL) author: John Snow Labs name: redl_oncology_biobert_wip date: 2023-01-15 tags: [licensed, clinical, oncology, en, relation_extraction, temporal, test, biomarker, anatomy, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model identifies relations between dates and other clinical entities, between tumor mentions and their size, between anatomical entities and other clinical entities, and between tests and their results. In contrast to re_oncology_granular, all these relation types are labeled as is_related_to. The different types of relations can be identified considering the pairs of entities that are linked. ## Predicted Entities `is_related_to` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biobert_wip_en_4.2.4_3.0_1673763869198.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_biobert_wip_en_4.2.4_3.0_1673763869198.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ | relation| entity1|entity1_begin|entity1_end| chunk1| entity2|entity2_begin|entity2_end| chunk2|confidence| +-------------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ |is_related_to|Cancer_Surgery| 2| 11|mastectomy|Relative_Date| 27| 40|two months ago|0.91422147| |is_related_to| Tumor_Size| 49| 52| 3 cm|Tumor_Finding| 54| 57| mass|0.90398973| +-------------+--------------+-------------+-----------+----------+-------------+-------------+-----------+--------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_biobert_wip| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.82 0.89 0.86 is_related_to 0.90 0.84 0.87 macro-avg 0.86 0.87 0.86 ``` --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_de_4.2.0_3.0_1664118903970.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42_de_4.2.0_3.0_1664118903970.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_xls_r_accent_germany_8_austria_2_s42| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Legal Interests Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_interests_bert date: 2023-03-05 tags: [en, legal, classification, clauses, interests, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Interests` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Interests`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_interests_bert_en_1.0.0_3.0_1678049881866.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_interests_bert_en_1.0.0_3.0_1678049881866.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_interests_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Interests]| |[Other]| |[Other]| |[Interests]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_interests_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Interests 0.97 0.93 0.95 40 Other 0.95 0.98 0.97 58 accuracy - - 0.96 98 macro-avg 0.96 0.95 0.96 98 weighted-avg 0.96 0.96 0.96 98 ``` --- layout: model title: English image_classifier_vit_lung_cancer ViTForImageClassification from Giuliano author: John Snow Labs name: image_classifier_vit_lung_cancer date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_lung_cancer` is a English model originally trained by Giuliano. ## Predicted Entities `LUAD`, `LUSC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lung_cancer_en_4.1.0_3.0_1660171240836.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_lung_cancer_en_4.1.0_3.0_1660171240836.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_lung_cancer", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_lung_cancer", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_lung_cancer| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Stop Words Cleaner for Swedish author: John Snow Labs name: stopwords_sv date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: sv edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, sv] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_sv_sv_2.5.4_2.4_1594742438273.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_sv_sv_2.5.4_2.4_1594742438273.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_sv", "sv") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Förutom att vara kungen i norr är John Snow en engelsk läkare och en ledare inom utveckling av anestesi och medicinsk hygien.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_sv", "sv") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Förutom att vara kungen i norr är John Snow en engelsk läkare och en ledare inom utveckling av anestesi och medicinsk hygien.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Förutom att vara kungen i norr är John Snow en engelsk läkare och en ledare inom utveckling av anestesi och medicinsk hygien."""] stopword_df = nlu.load('sv.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=6, result='Förutom', metadata={'sentence': '0'}), Row(annotatorType='token', begin=17, end=22, result='kungen', metadata={'sentence': '0'}), Row(annotatorType='token', begin=26, end=29, result='norr', metadata={'sentence': '0'}), Row(annotatorType='token', begin=31, end=32, result='är', metadata={'sentence': '0'}), Row(annotatorType='token', begin=34, end=37, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_sv| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|sv| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Mapping Abbreviations and Acronyms of Medical Regulatory Activities with Their Definitions author: John Snow Labs name: abbreviation_mapper date: 2022-05-11 tags: [en, abbreviation, definition, licensed, clinical, chunk_mapper] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps abbreviations and acronyms of medical regulatory activities with their `definition`. ## Predicted Entities `definition` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/abbreviation_mapper_en_3.5.1_3.0_1652307379928.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/abbreviation_mapper_en_3.5.1_3.0_1652307379928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") #NER model to detect abbreviations in the text abbr_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("abbr_ner") abbr_converter = NerConverter() \ .setInputCols(["sentence", "token", "abbr_ner"]) \ .setOutputCol("abbr_ner_chunk")\ chunkerMapper = ChunkMapperModel.pretrained("abbreviation_mapper", "en", "clinical/models")\ .setInputCols(["abbr_ner_chunk"])\ .setOutputCol("mappings")\ .setRel("definition") pipeline = Pipeline().setStages([document_assembler, sentence_detector, tokenizer, word_embeddings, abbr_ner, abbr_converter, chunkerMapper]) text = ["""Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet."""] data = spark.createDataFrame([text]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val abbr_ner = MedicalNerModel.pretrained("ner_abbreviation_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("abbr_ner") val abbr_converter = NerConverter() .setInputCols(Array("sentence", "token", "abbr_ner")) .setOutputCol("abbr_ner_chunk") val chunkerMapper = ChunkMapperModel.pretrained("abbreviation_mapper", "en", "clinical/models") .setInputCols(Array("abbr_ner_chunk")) .setOutputCol("mappings") .setRel("definition") val pipeline = new Pipeline().setStages(Array( document_assembler, sentence_detector, tokenizer, word_embeddings, abbr_ner, abbr_converter, chunkerMapper)) val test_sentence = """Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.""" val data = Seq(test_sentence).toDS.toDF("text") val result= pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.abbreviation_to_definition").predict("""Gravid with estimated fetal weight of 6-6/12 pounds. LABORATORY DATA: Laboratory tests include a CBC which is normal. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.""") ```

## Results ```bash +----------+------------------------------+ |ner_chunk |mapping_result | +----------+------------------------------+ |CBC |complete blood count | |HIV |human immunodeficiency virus | +----------+------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|abbreviation_mapper| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[abbr_ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|214.8 KB| ## References https://www.johnsnowlabs.com/marketplace/list-of-abbreviations-and-acronyms-for-medical-regulatory-activities/ --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from SSarim) author: John Snow Labs name: distilbert_qa_ssarim_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `SSarim`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ssarim_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769154633.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ssarim_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769154633.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ssarim_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ssarim_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ssarim_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SSarim/distilbert-base-uncased-finetuned-squad --- layout: model title: Abkhazian asr_wav2vec2_xls_r_300m_ab_CV8 TFWav2Vec2ForCTC from emre author: John Snow Labs name: asr_wav2vec2_xls_r_300m_ab_CV8 date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_xls_r_300m_ab_CV8` is a Abkhazian model originally trained by emre. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_xls_r_300m_ab_CV8_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_ab_CV8_ab_4.2.0_3.0_1664020050816.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_xls_r_300m_ab_CV8_ab_4.2.0_3.0_1664020050816.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_xls_r_300m_ab_CV8", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_xls_r_300m_ab_CV8", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_xls_r_300m_ab_CV8| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|1.2 GB| --- layout: model title: English BertForQuestionAnswering model (from ricardo-filho) author: John Snow Labs name: bert_qa_bert_large_faquad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_large_faquad` is a English model orginally trained by `ricardo-filho`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_faquad_en_4.0.0_3.0_1654185123808.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_faquad_en_4.0.0_3.0_1654185123808.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_faquad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_faquad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.large.by_ricardo-filho").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_faquad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ricardo-filho/bert_large_faquad --- layout: model title: Fast Neural Machine Translation Model from English to Rundi author: John Snow Labs name: opus_mt_en_run date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, run, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `run` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_run_xx_2.7.0_2.4_1609166290734.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_run_xx_2.7.0_2.4_1609166290734.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_run| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Arabic to Turkish author: John Snow Labs name: opus_mt_ar_tr date: 2021-06-01 tags: [open_source, seq2seq, translation, ar, tr, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: ar target languages: tr {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ar_tr_xx_3.1.0_2.4_1622563228141.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ar_tr_xx_3.1.0_2.4_1622563228141.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ar_tr| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Japanese Bert Embeddings (Base, Whole Word Masking) author: John Snow Labs name: bert_embeddings_bert_base_japanese_whole_word_masking date: 2022-04-11 tags: [bert, embeddings, ja, open_source] task: Embeddings language: ja edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-japanese-whole-word-masking` is a Japanese model orginally trained by `cl-tohoku`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_whole_word_masking_ja_3.4.2_3.0_1649674234386.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_japanese_whole_word_masking_ja_3.4.2_3.0_1649674234386.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_japanese_whole_word_masking| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ja| |Size:|415.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking - https://github.com/google-research/bert - https://github.com/cl-tohoku/bert-japanese/tree/v1.0 - https://github.com/attardi/wikiextractor - https://taku910.github.io/mecab/ - https://creativecommons.org/licenses/by-sa/3.0/ - https://www.tensorflow.org/tfrc/ --- layout: model title: English asr_wav2vec2_base_timit_demo_colab9 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab9 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab9` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab9_en_4.2.0_3.0_1664019968511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab9_en_4.2.0_3.0_1664019968511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab9', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab9", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Multilingual DistilBertForQuestionAnswering Cased model (from ZYW) author: John Snow Labs name: distilbert_qa_test_squad_trained date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `test-squad-trained` is a Multilingual model originally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_test_squad_trained_xx_4.3.0_3.0_1672775713238.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_test_squad_trained_xx_4.3.0_3.0_1672775713238.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_test_squad_trained","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_test_squad_trained","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_test_squad_trained| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/test-squad-trained --- layout: model title: Relation Extraction between Test and Results (ReDL) author: John Snow Labs name: redl_oncology_test_result_biobert_wip date: 2022-09-29 tags: [licensed, clinical, oncology, en, relation_extraction, test] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This relation extraction model links test extractions to their corresponding results. ## Predicted Entities `is_finding_of`, `O` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_oncology_test_result_biobert_wip_en_4.1.0_3.0_1664460623630.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_oncology_test_result_biobert_wip_en_4.1.0_3.0_1664460623630.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Each relevant relation pair in the pipeline should include one test entity (such as Biomarker, Imaging_Test, Pathology_Test or Oncogene) and one result entity (such as Biomarker_Result, Pathology_Result or Tumor_Finding).

## Results ```bash | chunk1 | entity1 | chunk2 | entity2 | relation | confidence | |-------- |----------------- |----------------------- |---------- |-------------- |----------- | |positive | Biomarker_Result | estrogen | Biomarker | is_finding_of | 0.99451536 | |positive | Biomarker_Result | progesterone receptors | Biomarker | is_finding_of | 0.99218905 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_oncology_test_result_biobert_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label recall precision f1 O 0.87 0.92 0.9 is_finding_of 0.93 0.88 0.9 macro-avg 0.90 0.90 0.9 ``` --- layout: model title: English BertForQuestionAnswering model (from ZYW) author: John Snow Labs name: bert_qa_squad_mbert_model_2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-mbert-model_2` is a English model orginally trained by `ZYW`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad_mbert_model_2_en_4.0.0_3.0_1654192072821.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad_mbert_model_2_en_4.0.0_3.0_1654192072821.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad_mbert_model_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad_mbert_model_2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.multi_lingual_bert.v2.by_ZYW").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad_mbert_model_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ZYW/squad-mbert-model_2 --- layout: model title: Bashkir asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt TFWav2Vec2ForCTC from AigizK author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt date: 2022-09-24 tags: [wav2vec2, ba, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: ba edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt` is a Bashkir model originally trained by AigizK. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_ba_4.2.0_3.0_1664040387674.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt_ba_4.2.0_3.0_1664040387674.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt', lang = 'ba') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt", lang = "ba") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_bashkir_cv7_opt| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|ba| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from hark99) author: John Snow Labs name: distilbert_qa_hark99_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `hark99`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_hark99_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771083025.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_hark99_base_uncased_finetuned_squad_en_4.3.0_3.0_1672771083025.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hark99_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_hark99_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_hark99_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/hark99/distilbert-base-uncased-finetuned-squad --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_test TFWav2Vec2ForCTC from ying-tina author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_test date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_test` is a English model originally trained by ying-tina. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_test_en_4.2.0_3.0_1664111990571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_test_en_4.2.0_3.0_1664111990571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_test', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_test", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_test| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Fast Neural Machine Translation Model from English to Papiamento author: John Snow Labs name: opus_mt_en_pap date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, pap, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `pap` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_pap_xx_2.7.0_2.4_1609169563076.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_pap_xx_2.7.0_2.4_1609169563076.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_pap| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nl32 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nl32` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl32_en_4.3.0_3.0_1675123928277.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl32_en_4.3.0_3.0_1675123928277.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nl32","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nl32","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nl32| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|145.4 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nl32 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Chinese Bert Embeddings (Bert, Whole Word Masking) author: John Snow Labs name: bert_embeddings_chinese_bert_wwm_ext date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `chinese-bert-wwm-ext` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_bert_wwm_ext_zh_3.4.2_3.0_1649668880514.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_bert_wwm_ext_zh_3.4.2_3.0_1649668880514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_bert_wwm_ext| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|384.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/hfl/chinese-bert-wwm-ext - https://arxiv.org/abs/1906.08101 - https://github.com/google-research/bert - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/MacBERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/HFL-Anthology - https://arxiv.org/abs/2004.13922 - https://arxiv.org/abs/1906.08101 --- layout: model title: Named Entity Recognition Profiling (Clinical) author: John Snow Labs name: ner_profiling_clinical date: 2023-04-28 tags: [licensed, en, clinical, profiling, ner_profiling, ner] task: [Named Entity Recognition, Pipeline Healthcare] language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `embeddings_clinical`. It has been updated by adding new clinical NER models and NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `jsl_ner_wip_clinical`,`jsl_ner_wip_greedy_clinical`,`jsl_ner_wip_modifier_clinical`, `jsl_rd_ner_wip_greedy_clinical`, `ner_abbreviation_clinical`, `ner_ade_binary`, `ner_ade_clinical`, `ner_anatomy`, `ner_anatomy_coarse`, `ner_bacterial_species`, `ner_biomarker`, `ner_biomedical_bc2gm`, `ner_bionlp`, `ner_cancer_genetics`, `ner_cellular`, `ner_chemd_clinical`, `ner_chemicals`, `ner_chemprot_clinical`, `ner_chexpert`, `ner_clinical`, `ner_clinical_large`, `ner_clinical_trials_abstracts`, `ner_covid_trials`, `ner_deid_augmented`, `ner_deid_enriched`, `ner_deid_generic_augmented`,`ner_deid_large`, `ner_deid_sd`,`ner_deid_sd_large`,`ner_deid_subentity_augmented`,`ner_deid_subentity_augmented_i2b2`, `ner_deid_synthetic`, `ner_diseases`, `ner_diseases_large`, `ner_drugprot_clinical`, `ner_drugs`, `ner_drugs_greedy`, `ner_drugs_large`, `ner_eu_clinical_case`, `ner_eu_clinical_condition`, `ner_events_admission_clinical`, `ner_events_clinical`, `ner_financial_contract`, `ner_genetic_variants`, `ner_healthcare`, `ner_human_phenotype_gene_clinical`, `ner_human_phenotype_go_clinical`, `ner_jsl`, `ner_jsl_enriched`, `ner_jsl_slim`, `ner_living_species`, `ner_measurements_clinical`, `ner_medmentions_coarse`, `ner_nature_nero_clinical`, `ner_nihss`, `ner_oncology`, `ner_oncology_anatomy_general`, `ner_oncology_anatomy_granular`, `ner_oncology_biomarker`, `ner_oncology_demographics`, `ner_oncology_diagnosis`, `ner_oncology_posology`, `ner_oncology_response_to_treatment`, `ner_oncology_test`, `ner_oncology_therapy`, `ner_oncology_tnm`, `ner_oncology_unspecific_posology`, `ner_oncology_wip`, `ner_pathogen`, `ner_posology`, `ner_posology_experimental`, `ner_posology_greedy`, `ner_posology_large`, `ner_posology_small`, `ner_radiology`, `ner_radiology_wip_clinical`, `ner_risk_factors`, `ner_sdoh_access_to_healthcare_wip`, `ner_sdoh_community_condition_wip`, `ner_sdoh_demographics_wip`, `ner_sdoh_health_behaviours_problems_wip`, `ner_sdoh_income_social_status_wip`, `ner_sdoh_mentions`, `ner_sdoh_slim_wip`, `ner_sdoh_social_environment_wip`, `ner_sdoh_substance_usage_wip`, `ner_sdoh_wip`, `ner_supplement_clinical`, `ner_vop_anatomy_wip`, `ner_vop_clinical_dept_wip`, `ner_vop_demographic_wip`, `ner_vop_problem_reduced_wip`, `ner_vop_problem_wip`, `ner_vop_slim_wip`, `ner_vop_temporal_wip`, `ner_vop_test_wip`, `ner_vop_treatment_wip`, `ner_vop_wip`, `nerdl_tumour_demo` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.4.0_3.0_1682686984113.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_clinical_en_4.4.0_3.0_1682686984113.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline("ner_profiling_clinical", "en", "clinical/models") result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline("ner_profiling_clinical", "en", "clinical/models") val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_clinical").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting.""") ```

## Results ```bash ******************** ner_jsl Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('subsequent', 'Modifier'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Diabetes'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Communicable_Disease'), ('obesity', 'Obesity'), ('body mass index', 'Symptom'), ('33.5 kg/m2', 'Weight'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_diseases_large Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('vomiting', 'Disease')] ******************** ner_radiology Model Results ******************** [('gestational diabetes mellitus', 'Disease_Syndrome_Disorder'), ('type two diabetes mellitus', 'Disease_Syndrome_Disorder'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('acute hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Disease_Syndrome_Disorder'), ('body', 'BodyPart'), ('mass index', 'Symptom'), ('BMI', 'Test'), ('33.5', 'Measurements'), ('kg/m2', 'Units'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus', 'PROBLEM'), ('T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'PROBLEM'), ('BMI', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_medmentions_coarse Model Results ******************** [('female', 'Organism_Attribute'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('diabetes mellitus', 'Disease_or_Syndrome'), ('T2DM', 'Disease_or_Syndrome'), ('HTG-induced pancreatitis', 'Disease_or_Syndrome'), ('associated with', 'Qualitative_Concept'), ('acute hepatitis', 'Disease_or_Syndrome'), ('obesity', 'Disease_or_Syndrome'), ('body mass index', 'Clinical_Attribute'), ('BMI', 'Clinical_Attribute'), ('polyuria', 'Sign_or_Symptom'), ('polydipsia', 'Sign_or_Symptom'), ('poor appetite', 'Sign_or_Symptom'), ('vomiting', 'Sign_or_Symptom')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_clinical| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|3.1 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: Multilingual T5ForConditionalGeneration Cased model (from qiaoyi) author: John Snow Labs name: t5_comment_summarization4designtutor date: 2023-01-30 tags: [ro, fr, de, en, open_source, t5, xx] task: Text Generation language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Comment_Summarization4DesignTutor` is a Multilingual model originally trained by `qiaoyi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_comment_summarization4designtutor_xx_4.3.0_3.0_1675096380129.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_comment_summarization4designtutor_xx_4.3.0_3.0_1675096380129.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_comment_summarization4designtutor","xx") \ .setInputCols(["document"]) \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_comment_summarization4designtutor","xx") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_comment_summarization4designtutor| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|xx| |Size:|271.8 MB| ## References - https://huggingface.co/qiaoyi/Comment_Summarization4DesignTutor - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/1805.12471 - https://nlp.stanford.edu/~socherr/EMNLP2013_RNTN.pdf - https://aclanthology.org/I05-5002 - https://arxiv.org/abs/1708.00055 - https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs - https://arxiv.org/abs/1704.05426 - https://arxiv.org/abs/1606.05250 - https://link.springer.com/chapter/10.1007/11736790_9 - https://semanticsarchive.net/Archive/Tg3ZGI2M/Marneffe.pdf - https://www.researchgate.net/publication/221251392_Choice_of_Plausible_Alternatives_An_Evaluation_of_Commonsense_Causal_Reasoning - https://arxiv.org/abs/1808.09121 - https://aclanthology.org/N18-1023 - https://arxiv.org/abs/1810.12885 - https://arxiv.org/abs/1905.10044 - https://arxiv.org/pdf/1910.10683.pdf - https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67 --- layout: model title: Detect Bacterial Species (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_bacteria date: 2022-01-07 tags: [bacteria, bertfortokenclassification, ner, en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 2.4 supported: true annotator: MedicalBertForTokenClassifier article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect different types of species of bacteria in text using pretrained NER model. This model is trained with the `BertForTokenClassification` method from `transformers` library and imported into Spark NLP. ## Predicted Entities `SPECIES` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_en_3.3.4_2.4_1641568604267.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bacteria_en_3.3.4_2.4_1641568604267.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl22","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl22","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl22| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|373.8 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl22 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Extract Temporal Entities from Voice of the Patient Documents (embeddings_clinical_large) author: John Snow Labs name: ner_vop_temporal_emb_clinical_large_final date: 2023-06-06 tags: [clinical, licensed, ner, en, vop, temporal] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts temporal references from the documents transferred from the patient’s own sentences. ## Predicted Entities `DateTime`, `Duration`, `Frequency` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_temporal_emb_clinical_large_final_en_4.4.3_3.0_1686076327359.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_temporal_emb_clinical_large_final_en_4.4.3_3.0_1686076327359.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:-----------|:------------| | last month | DateTime | | yesterday | DateTime | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_temporal_emb_clinical_large_final| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_large| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 DateTime 4021 585 381 4402 0.87 0.91 0.89 Duration 1889 266 421 2310 0.88 0.82 0.85 Frequency 890 166 189 1079 0.84 0.82 0.83 macro_avg 6800 1017 991 7791 0.86 0.85 0.86 micro_avg 6800 1017 991 7791 0.87 0.87 0.87 ``` --- layout: model title: Italian BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_italian_finedtuned_squadv1_it_alfa date: 2022-06-06 tags: [it, open_source, question_answering, bert] task: Question Answering language: it edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-italian-finedtuned-squadv1-it-alfa` is a Italian model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_italian_finedtuned_squadv1_it_alfa_it_4.0.0_3.0_1654536030553.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_italian_finedtuned_squadv1_it_alfa_it_4.0.0_3.0_1654536030553.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_italian_finedtuned_squadv1_it_alfa","it") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_italian_finedtuned_squadv1_it_alfa","it") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("it.answer_question.squad.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_italian_finedtuned_squadv1_it_alfa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|it| |Size:|410.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-italian-finedtuned-squadv1-it-alfa - https://github.com/crux82/squad-it/blob/master/README.md#evaluating-a-neural-model-over-squad-it - https://twitter.com/mrm8488 - https://link.springer.com/chapter/10.1007/978-3-030-03840-3_29 - https://github.com/crux82/squad-it - https://rajpurkar.github.io/SQuAD-explorer/ - https://www.linkedin.com/in/manuel-romero-cs/ --- layout: model title: Pipeline to Summarize Clinical Notes author: John Snow Labs name: summarizer_clinical_jsl_pipeline date: 2023-05-29 tags: [licensed, en, clinical, text_summarization] task: Summarization language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [summarizer_clinical_jsl](https://nlp.johnsnowlabs.com/2023/03/25/summarizer_clinical_jsl.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_pipeline_en_4.4.2_3.0_1685392074544.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_pipeline_en_4.4.2_3.0_1685392074544.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("summarizer_clinical_jsl_pipeline", "en", "clinical/models") text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("summarizer_clinical_jsl_pipeline", "en", "clinical/models") val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash. """ val result = pipeline.fullAnnotate(text) ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_fine_tuned_model","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_fine_tuned_model","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_fine_tuned_model| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|918.7 MB| ## References - https://huggingface.co/marcus2000/fine_tuned_t5_model - https://paperswithcode.com/sota?task=automatic-speech-recognition&dataset=Librispeech+%28clean%29 --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_ner_778023879 date: 2023-03-14 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-ner-778023879` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1678783208077.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1678783208077.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_ner_778023879| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-ner-778023879 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_128 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_128_zh_4.2.4_3.0_1670021654446.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_128_zh_4.2.4_3.0_1670021654446.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_128| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|13.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-128 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Legal NER for MAPA(Multilingual Anonymisation for Public Administrations) author: John Snow Labs name: legner_mapa date: 2023-04-27 tags: [es, licensed, legal, ner, mapa] task: Named Entity Recognition language: es edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The dataset consists of 12 documents taken from EUR-Lex, a multilingual corpus of court decisions and legal dispositions in the 24 official languages of the European Union. This model extracts `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, and `PERSON` entities from `Spanish` documents. ## Predicted Entities `ADDRESS`, `AMOUNT`, `DATE`, `ORGANISATION`, `PERSON` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_mapa_es_1.0.0_3.0_1682593085140.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_mapa_es_1.0.0_3.0_1682593085140.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_base_es_cased", "es")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_mapa", "es", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Heiko Jonny Maniero , de nacionalidad italiana , nació y reside en Alemania."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-------------------+---------+ |chunk |ner_label| +-------------------+---------+ |Heiko Jonny Maniero|PERSON | |Alemania |ADDRESS | +-------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_mapa| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Size:|1.4 MB| ## References The dataset is available [here](https://huggingface.co/datasets/joelito/mapa). ## Benchmarking ```bash label precision recall f1-score support ADDRESS 1.00 0.86 0.92 7 AMOUNT 1.00 1.00 1.00 1 DATE 1.00 0.92 0.96 24 ORGANISATION 0.83 0.71 0.77 7 PERSON 0.75 0.71 0.73 17 macro-avg 0.90 0.82 0.86 56 macro-avg 0.92 0.84 0.88 56 weighted-avg 0.90 0.82 0.86 56 ``` --- layout: model title: GloVe Embeddings 6B 300 (Multilingual) author: John Snow Labs name: glove_6B_300 date: 2020-01-22 task: Embeddings language: xx edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [open_source, embeddings] supported: true annotator: WordEmbeddingsModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description GloVe (Global Vectors) is a model for distributed word representation. This is achieved by mapping words into a meaningful space where the distance between words is related to semantic similarity. It outperformed many common Word2vec models on the word analogy task. One benefit of GloVe is that it is the result of directly modeling relationships, instead of getting them as a side effect of training a language model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/glove_6B_300_xx_2.4.0_2.4_1579698630432.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/glove_6B_300_xx_2.4.0_2.4_1579698630432.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddings.pretrained("glove_6B_300", "xx") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love Spark NLP']], ["text"])) ``` ```scala val embeddings = WordEmbeddings.pretrained("glove_6B_300", "xx") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""I love Spark NLP"""] glove_df = nlu.load('xx.embed.glove.6B_300').predict(text) glove_df ```

{:.h2_title} ## Results ```bash token | glove_embeddings | -------|----------------------------------------------------| I | [0.1941000074148178, 0.22603000700473785, -0.4...] | love | [0.13948999345302582, 0.534529983997345, -0.25...] | Spark | [0.20353999733924866, 0.6292600035667419, 0.27...] | NLP | [0.059436000883579254, 0.18411000072956085, -0...] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|glove_6B_300| |Type:|embeddings| |Compatibility:|Spark NLP 2.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[xx]| |Dimension:|300| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from [https://nlp.stanford.edu/projects/glove/](https://nlp.stanford.edu/projects/glove/) --- layout: model title: English DistilBertForQuestionAnswering Cased model (from ajaypyatha) author: John Snow Labs name: distilbert_qa_sdsqna date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sdsqna` is a English model originally trained by `ajaypyatha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_sdsqna_en_4.3.0_3.0_1672775452508.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_sdsqna_en_4.3.0_3.0_1672775452508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sdsqna","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_sdsqna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_sdsqna| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ajaypyatha/sdsqna --- layout: model title: Oncology Pipeline for Diagnosis Entities author: John Snow Labs name: oncology_diagnosis_pipeline date: 2023-03-29 tags: [licensed, pipeline, oncology, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.2 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline includes Named-Entity Recognition, Assertion Status, Relation Extraction and Entity Resolution models to extract information from oncology texts. This pipeline focuses on entities related to oncological diagnosis. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/oncology_diagnosis_pipeline_en_4.3.2_3.2_1680111732253.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/oncology_diagnosis_pipeline_en_4.3.2_3.2_1680111732253.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("oncology_diagnosis_pipeline", "en", "clinical/models") text = '''Two years ago, the patient presented with a 4-cm tumor in her left breast. She was diagnosed with ductal carcinoma. According to her last CT, she has no lung metastases.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("oncology_diagnosis_pipeline", "en", "clinical/models") val text = "Two years ago, the patient presented with a 4-cm tumor in her left breast. She was diagnosed with ductal carcinoma. According to her last CT, she has no lung metastases." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.oncology_diagnosis.pipeline").predict("""Two years ago, the patient presented with a 4-cm tumor in her left breast. She was diagnosed with ductal carcinoma. According to her last CT, she has no lung metastases.""") ```

## Results ```bash " ******************** ner_oncology_wip results ******************** | chunk | ner_label | |:-----------|:------------------| | 4-cm | Tumor_Size | | tumor | Tumor_Finding | | left | Direction | | breast | Site_Breast | | ductal | Histological_Type | | carcinoma | Cancer_Dx | | lung | Site_Lung | | metastases | Metastasis | ******************** ner_oncology_diagnosis_wip results ******************** | chunk | ner_label | |:-----------|:------------------| | 4-cm | Tumor_Size | | tumor | Tumor_Finding | | ductal | Histological_Type | | carcinoma | Cancer_Dx | | metastases | Metastasis | ******************** ner_oncology_tnm_wip results ******************** | chunk | ner_label | |:-----------|:------------------| | 4-cm | Tumor_Description | | tumor | Tumor | | ductal | Tumor_Description | | carcinoma | Cancer_Dx | | metastases | Metastasis | ******************** assertion_oncology_wip results ******************** | chunk | ner_label | assertion | |:-----------|:------------------|:------------| | tumor | Tumor_Finding | Present | | ductal | Histological_Type | Present | | carcinoma | Cancer_Dx | Present | | metastases | Metastasis | Absent | ******************** assertion_oncology_problem_wip results ******************** | chunk | ner_label | assertion | |:-----------|:------------------|:-----------------------| | tumor | Tumor_Finding | Medical_History | | ductal | Histological_Type | Medical_History | | carcinoma | Cancer_Dx | Medical_History | | metastases | Metastasis | Hypothetical_Or_Absent | ******************** re_oncology_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------|:--------------|:-----------|:--------------|:--------------| | 4-cm | Tumor_Size | tumor | Tumor_Finding | is_related_to | | 4-cm | Tumor_Size | carcinoma | Cancer_Dx | O | | tumor | Tumor_Finding | breast | Site_Breast | is_related_to | | breast | Site_Breast | carcinoma | Cancer_Dx | O | | lung | Site_Lung | metastases | Metastasis | is_related_to | ******************** re_oncology_granular_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------|:--------------|:-----------|:--------------|:---------------| | 4-cm | Tumor_Size | tumor | Tumor_Finding | is_size_of | | 4-cm | Tumor_Size | carcinoma | Cancer_Dx | O | | tumor | Tumor_Finding | breast | Site_Breast | is_location_of | | breast | Site_Breast | carcinoma | Cancer_Dx | O | | lung | Site_Lung | metastases | Metastasis | is_location_of | ******************** re_oncology_size_wip results ******************** | chunk1 | entity1 | chunk2 | entity2 | relation | |:---------|:-----------|:----------|:--------------|:-----------| | 4-cm | Tumor_Size | tumor | Tumor_Finding | is_size_of | | 4-cm | Tumor_Size | carcinoma | Cancer_Dx | O | ******************** ICD-O resolver results ******************** | chunk | ner_label | code | normalized_term | |:-----------|:------------------|:-------|:------------------| | tumor | Tumor_Finding | 8000/1 | tumor | | breast | Site_Breast | C50 | breast | | ductal | Histological_Type | 8500/2 | dcis | | carcinoma | Cancer_Dx | 8010/3 | carcinoma | | lung | Site_Lung | C34.9 | lung | | metastases | Metastasis | 8000/6 | tumor, metastatic | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|oncology_diagnosis_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|2.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - ChunkMergeModel - ChunkMergeModel - AssertionDLModel - AssertionDLModel - PerceptronModel - DependencyParserModel - RelationExtractionModel - RelationExtractionModel - RelationExtractionModel - ChunkMergeModel - Chunk2Doc - BertSentenceEmbeddings - SentenceEntityResolverModel --- layout: model title: Deidentify PHI (Large) author: John Snow Labs name: deidentify_large language: en nav_key: models repository: clinical/models date: 2020-08-04 task: De-identification edition: Healthcare NLP 2.5.5 spark_version: 2.4 tags: [deidentify, en, clinical, licensed] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Deidentify (Large) is a deidentification model. It identifies instances of protected health information in text documents, and it can either obfuscate them (e.g., replacing names with different, fake names) or mask them (e.g., replacing "2020-06-04" with "<DATE>"). This model is useful for maintaining HIPAA compliance when dealing with text documents that contain protected health information. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/DEID_PHI_TEXT){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/DEID_PHI_TEXT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/nerdl_deid_en_1.8.0_2.4_1545462443516.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/nerdl_deid_en_1.8.0_2.4_1545462443516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... nlp_pipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter]) result = nlp_pipeline.transform(spark.createDataFrame([['Patient AIQING, 25 month years-old , born in Beijing, was transfered to the The Johns Hopkins Hospital. Phone number: (541) 754-3010. MSW 100009632582 for his colonic polyps. He wants to know the results from them. He is not taking hydrochlorothiazide and is curious about his blood pressure. He said he has cut his alcohol back to 6 pack once a week. He has cut back his cigarettes to one time per week. P: Follow up with Dr. Hobbs in 3 months. Gilbert P. Perez, M.D.']], ["text"])) obfuscation = DeIdentificationModel.pretrained("deidentify_large", "en", "clinical/models") \ .setInputCols(["sentence", "token", "ner_chunk"]) \ .setOutputCol("obfuscated") \ .setMode("obfuscate") deid_text = obfuscation.transform(result) ``` ```scala ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("Patient AIQING, 25 month years-old , born in Beijing, was transfered to the The Johns Hopkins Hospital. Phone number: (541) 754-3010. MSW 100009632582 for his colonic polyps. He wants to know the results from them. He is not taking hydrochlorothiazide and is curious about his blood pressure. He said he has cut his alcohol back to 6 pack once a week. He has cut back his cigarettes to one time per week. P: Follow up with Dr. Hobbs in 3 months. Gilbert P. Perez, M.D.").toDF("text") val result = pipeline.fit(data).transform(data) val deid = DeIdentificationModel.pretrained("deidentify_large", "en", "clinical/models") .setInputCols(Array("sentence", "token", "ner_chunk")) .setOutputCol("obfuscated") .setMode("obfuscate") val deid_text = new deid.transform(result) ``` {:.nlu-block} ```python import nlu nlu.load("en.de_identify.large").predict("""Patient AIQING, 25 month years-old , born in Beijing, was transfered to the The Johns Hopkins Hospital. Phone number: (541) 754-3010. MSW 100009632582 for his colonic polyps. He wants to know the results from them. He is not taking hydrochlorothiazide and is curious about his blood pressure. He said he has cut his alcohol back to 6 pack once a week. He has cut back his cigarettes to one time per week. P: Follow up with Dr. Hobbs in 3 months. Gilbert P. Perez, M.D.""") ```

{:.h2_title} ## Results ```bash | | sentence | deidentified | |--:|--------------------------------------------------:|--------------------------------------------------:| | 0 | Patient AIQING, 25 month years-old , born in B... | Patient CAM, month years-old , born in M... | | 1 | Phone number: (541) 754-3010. | Phone number: (603)531-7148. | | 2 | MSW 100009632582 for his colonic polyps. | MSW for his colonic polyps. | | 3 | He wants to know the results from them. | He wants to know the results from them. | | 4 | He is not taking hydrochlorothiazide and is cu... | He is not taking hydrochlorothiazide and is cu... | | 5 | He said he has cut his alcohol back to 6 pack ... | He said he has cut his alcohol back to 6 pack ... | | 6 | He \nhas cut back his cigarettes to one time p... | He \nhas cut back his cigarettes to one time p... | | 7 | P: Follow up with Dr. Hobbs in 3 months. | P: Follow up with Dr. RODOLPH in 3 months. | | 8 | Gilbert P. Perez, M.D. | Gertie, M.D. | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deidentify_large| |Type:|deid| |Compatibility:| Healthcare NLP 2.5.5| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, ner_chunk]| |Output Labels:|[obfuscated]| |Language:|en| |Case sensitive:|false| {:.h2_title} ## Data Source The model was trained based on data from [https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/](https://portal.dbmi.hms.harvard.edu/projects/n2c2-2014/) --- layout: model title: Modern Greek (1453-) asr_wav2vec2_large_xlsr_greek_2 TFWav2Vec2ForCTC from skylord author: John Snow Labs name: asr_wav2vec2_large_xlsr_greek_2 date: 2022-09-25 tags: [wav2vec2, el, audio, open_source, asr] task: Automatic Speech Recognition language: el edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_greek_2` is a Modern Greek (1453-) model originally trained by skylord. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_greek_2_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_greek_2_el_4.2.0_3.0_1664111950103.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_greek_2_el_4.2.0_3.0_1664111950103.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_greek_2", "el")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_greek_2", "el") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_greek_2| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|el| |Size:|1.2 GB| --- layout: model title: French CamemBert Embeddings (from peterhsu) author: John Snow Labs name: camembert_embeddings_tf_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tf-dummy-model` is a French model orginally trained by `peterhsu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_tf_generic_model_fr_3.4.4_3.0_1653992001210.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_tf_generic_model_fr_3.4.4_3.0_1653992001210.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_tf_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/peterhsu/tf-dummy-model --- layout: model title: English RobertaForQuestionAnswering (from tli8hf) author: John Snow Labs name: roberta_qa_unqover_roberta_base_newsqa date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `unqover-roberta-base-newsqa` is a English model originally trained by `tli8hf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_roberta_base_newsqa_en_4.0.0_3.0_1655740101329.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_unqover_roberta_base_newsqa_en_4.0.0_3.0_1655740101329.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_roberta_base_newsqa","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_unqover_roberta_base_newsqa","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.news.roberta.base").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_unqover_roberta_base_newsqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tli8hf/unqover-roberta-base-newsqa --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from ksabeh) author: John Snow Labs name: roberta_qa_base_attribute_correction date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-attribute-correction` is a English model originally trained by `ksabeh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_attribute_correction_en_4.3.0_3.0_1674212650491.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_attribute_correction_en_4.3.0_3.0_1674212650491.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_attribute_correction","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_attribute_correction","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_attribute_correction| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|430.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ksabeh/roberta-base-attribute-correction --- layout: model title: Legal Air And Space Transport Document Classifier (EURLEX) author: John Snow Labs name: legclf_air_and_space_transport_bert date: 2023-03-06 tags: [en, legal, classification, clauses, air_and_space_transport, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_air_and_space_transport_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Air_and_Space_Transport or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Air_and_Space_Transport`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_air_and_space_transport_bert_en_1.0.0_3.0_1678111618067.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_air_and_space_transport_bert_en_1.0.0_3.0_1678111618067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_air_and_space_transport_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Air_and_Space_Transport]| |[Other]| |[Other]| |[Air_and_Space_Transport]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_air_and_space_transport_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.6 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Air_and_Space_Transport 0.94 0.97 0.95 65 Other 0.97 0.93 0.95 60 accuracy - - 0.95 125 macro-avg 0.95 0.95 0.95 125 weighted-avg 0.95 0.95 0.95 125 ``` --- layout: model title: Multilingual RobertaForQuestionAnswering (from leolin12345) author: John Snow Labs name: roberta_qa_ft_lr_cu_leolin12345 date: 2022-06-21 tags: [cu, ft, lr, open_source, question_answering, roberta, xx] task: Question Answering language: xx edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ft-lr-cu` is a Multilingual model originally trained by `leolin12345`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ft_lr_cu_leolin12345_xx_4.0.0_3.0_1655789046666.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ft_lr_cu_leolin12345_xx_4.0.0_3.0_1655789046666.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_ft_lr_cu_leolin12345","xx") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_ft_lr_cu_leolin12345","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("xx.answer_question.roberta").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ft_lr_cu_leolin12345| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|xx| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/leolin12345/ft-lr-cu --- layout: model title: Sentence Entity Resolver for Billable ICD10-CM HCC Codes (sbiobertresolve_icd10cm_slim_billable_hcc) author: John Snow Labs name: sbiobertresolve_icd10cm_slim_billable_hcc date: 2023-05-31 tags: [licensed, en, clinical, icd10cm, entity_resolution] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted clinical entities to ICD-10-CM codes using `sbiobert_base_cased_mli` sentence bert embeddings. In this model, synonyms having low cosine similarity to unnormalized terms are dropped. It returns the official resolution text within the brackets and also provides billable and Hierarchical Condition Categories (HCC) information of the codes in `all_k_aux_labels` parameter in the metadata. This column can be divided to get further details: `billable status || hcc status || hcc score`. For example, if `all_k_aux_labels` is like `1||1||19` which means the `billable status` is 1, `hcc status` is 1, and `hcc score` is 19. ## Predicted Entities `ICD-10-CM Codes`, `billable status`, `hcc status`, `hcc score` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_ICD10_CM.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_4.4.2_3.0_1685500916240.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_slim_billable_hcc_en_4.4.2_3.0_1685500916240.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | ner_chunk| entity|icd10_code| resolutions| all_codes| hcc_list| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.4|[gestational diabetes mellitus [gestational diabetes mellitus], gestatio...|[O24.4, O24.41, Z86.32, O24.11, O24.81, P70.2, O24.01, O24.42, O24.414, ...|[0||0||0, 0||0||0, 1||0||0, 0||0||0, 0||0||0, 1||0||0, 0||0||0, 0||0||0,...| |subsequent type two diabetes mellitus|PROBLEM| E11|[type 2 diabetes mellitus [type 2 diabetes mellitus], type 2 diabetes me...|[E11, E11.62, E11.5, E11.69, E11.59, E09, E11.6, E11.8, E11.4, E11.628, ...|[0||0||0, 0||0||0, 0||0||0, 1||1||18, 1||1||18, 0||0||0, 0||0||0, 1||1||...| | obesity|PROBLEM| E66|[overweight and obesity [overweight and obesity], overweight [overweight...|[E66, E66.3, E66.8, E66.0, E66.1, E88.81, E66.09, E66.01, E34.4, E66.9, ...|[0||0||0, 1||0||0, 1||0||0, 0||0||0, 1||0||0, 1||0||0, 1||0||0, 1||1||22...| | a body mass index|PROBLEM| Z68|[body mass index [bmi] [body mass index [bmi]], localized adiposity [loc...|[Z68, E65, L02.221, Z96.81, Y92.81, Y93.75, L02.23, L02.22, M67.49, R73,...|[0||0||0, 1||0||0, 1||0||0, 1||0||0, 0||0||0, 1||0||0, 0||0||0, 0||0||0,...| | polyuria|PROBLEM| R35|[polyuria [polyuria], nocturnal polyuria [nocturnal polyuria], other pol...|[R35, R35.81, R35.89, R35.8, R31, R30.0, E72.01, R80, R34, R82.4, R82.99...|[0||0||0, 1||0||0, 1||0||0, 0||0||0, 0||0||0, 1||0||0, 1||1||23, 0||0||0...| | polydipsia|PROBLEM| R63.1|[polydipsia [polydipsia], polyhydramnios [polyhydramnios], parasomnia [p...|[R63.1, O40, G47.5, R63.2, R00.2, G47.1, G47.13, F51.11, G47.19, L68.3, ...|[1||0||0, 0||0||0, 0||0||0, 1||0||0, 1||0||0, 0||0||0, 1||0||0, 1||0||0,...| | vomiting|PROBLEM| R11.1|[vomiting [vomiting], cyclical vomiting [cyclical vomiting], nausea [nau...|[R11.1, G43.A, R11.0, R11, R11.14, R11.12, R23.1, G47.51, R11.10, H57.03...|[0||0||0, 0||0||0, 1||0||0, 0||0||0, 1||0||0, 1||0||0, 1||0||0, 1||0||0,...| | a respiratory tract infection|PROBLEM| T17|[foreign body in respiratory tract [foreign body in respiratory tract], ...|[T17, T81.4, T81.81, J95.851, T17.8, Z87.0, J44.0, J06, T81.44, Z22, T17...|[0||0||0, 0||0||0, 0||0||0, 1||1||114, 0||0||0, 0||0||0, 1||1||111, 0||0...| +-------------------------------------+-------+----------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_slim_billable_hcc| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|435.3 MB| |Case sensitive:|false| --- layout: model title: Chinese Bert Embeddings (Base, trained on Oscar dataset) author: John Snow Labs name: bert_embeddings_mengzi_oscar_base date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `mengzi-oscar-base` is a Chinese model orginally trained by `Langboat`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_mengzi_oscar_base_zh_3.4.2_3.0_1649670580702.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_mengzi_oscar_base_zh_3.4.2_3.0_1649670580702.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_mengzi_oscar_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|383.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/Langboat/mengzi-oscar-base - https://github.com/microsoft/Oscar - https://github.com/Langboat/Mengzi - https://arxiv.org/abs/2110.06696 - https://github.com/microsoft/Oscar/blob/master/INSTALL.md - https://github.com/Langboat/Mengzi/blob/main/Mengzi-Oscar.md --- layout: model title: Pipeline to Detect Drug Information author: John Snow Labs name: ner_posology_healthcare_pipeline date: 2022-03-22 tags: [licensed, ner, clinical, drug, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_posology_healthcare](https://nlp.johnsnowlabs.com/2021/04/01/ner_posology_healthcare_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_POSOLOGY.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_healthcare_pipeline_en_3.4.1_3.0_1647942842636.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_healthcare_pipeline_en_3.4.1_3.0_1647942842636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_posology_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate('The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.') ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_posology_healthcare_pipeline", "en", "clinical/models") pipeline.fullAnnotate("The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology.healthcare_pipeline").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunks |entities | +--------------+---------+ |insulin |Drug | |Bactrim |Drug | |for 14 days |Duration | |Fragmin |Drug | |5000 units |Dosage | |subcutaneously|Route | |daily |Frequency| |Xenaderm |Drug | |topically |Route | |b.i.d |Frequency| |Lantus |Drug | |40 units |Dosage | |subcutaneously|Route | |at bedtime |Frequency| |OxyContin |Drug | |30 mg |Strength | |p.o |Route | |q.12 h |Frequency| |folic acid |Drug | |1 mg |Strength | +--------------+---------+ only showing top 20 rows ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_posology_healthcare_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|513.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Translate Romance languages to English Pipeline author: John Snow Labs name: translate_roa_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, roa, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `roa` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_roa_en_xx_2.7.0_2.4_1609687168744.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_roa_en_xx_2.7.0_2.4_1609687168744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_roa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_roa_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.roa.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_roa_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForSequenceClassification Cased model (from Shenzy) author: John Snow Labs name: roberta_classifier_sentence_classification4designtutor date: 2022-12-09 tags: [en, open_source, roberta, sequence_classification, classification] task: Text Classification language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `Sentence_Classification4DesignTutor` is a English model originally trained by `Shenzy`. ## Predicted Entities `2`, `1`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_classifier_sentence_classification4designtutor_en_4.2.4_3.0_1670621867959.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_classifier_sentence_classification4designtutor_en_4.2.4_3.0_1670621867959.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_sentence_classification4designtutor","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, roberta_classifier]) data = spark.createDataFrame([["I love you!"], ["I feel lucky to be here."]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val roberta_classifier = RoBertaForSequenceClassification.pretrained("roberta_classifier_sentence_classification4designtutor","en") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, roberta_classifier)) val data = Seq("I love you!").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_classifier_sentence_classification4designtutor| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Shenzy/Sentence_Classification4DesignTutor --- layout: model title: Hindi BertForQuestionAnswering model (from bhavikardeshna) author: John Snow Labs name: bert_qa_multilingual_bert_base_cased_hindi date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: hi edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `multilingual-bert-base-cased-hindi` is a Hindi model orginally trained by `bhavikardeshna`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_multilingual_bert_base_cased_hindi_hi_4.0.0_3.0_1654188532153.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_multilingual_bert_base_cased_hindi_hi_4.0.0_3.0_1654188532153.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_multilingual_bert_base_cased_hindi","hi") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_multilingual_bert_base_cased_hindi","hi") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("hi.answer_question.bert.multilingual_hindi_tuned_base_cased.by_bhavikardeshna").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_multilingual_bert_base_cased_hindi| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hi| |Size:|665.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/bhavikardeshna/multilingual-bert-base-cased-hindi --- layout: model title: Word2Vec Embeddings in Catalan (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, ca, open_source] task: Embeddings language: ca edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ca_3.4.1_3.0_1647288979526.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ca_3.4.1_3.0_1647288979526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_economic_conditions_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Economic_Conditions]| |[Other]| |[Other]| |[Economic_Conditions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_economic_conditions_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Economic_Conditions 0.85 0.85 0.85 62 Other 0.84 0.84 0.84 56 accuracy - - 0.85 118 macro-avg 0.85 0.85 0.85 118 weighted-avg 0.85 0.85 0.85 118 ``` --- layout: model title: Legal Sales Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_sales_agreement_bert date: 2023-02-02 tags: [en, legal, classification, sales, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_sales_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `sales-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `sales-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_sales_agreement_bert_en_1.0.0_3.0_1675359477213.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_sales_agreement_bert_en_1.0.0_3.0_1675359477213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_sales_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[sales-agreement]| |[other]| |[other]| |[sales-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_sales_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.88 0.93 0.90 122 sales-agreement 0.84 0.76 0.80 63 accuracy - - 0.87 185 macro-avg 0.86 0.84 0.85 185 weighted-avg 0.87 0.87 0.87 185 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Shona author: John Snow Labs name: opus_mt_en_sn date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, sn, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `sn` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_sn_xx_2.7.0_2.4_1609170752195.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_sn_xx_2.7.0_2.4_1609170752195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_sn| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Financial News Summarization (Headers, Large) author: John Snow Labs name: finsum_news_headers_lg date: 2022-11-24 tags: [financial, summarization, summary, en, licensed] task: Summarization language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Financial news Summarizer, aimed to extract headers from financial news. This is a `lg` (large) version. Other versions may be found in Models Hub. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finsum_news_headers_lg_en_1.0.0_3.0_1669287223603.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finsum_news_headers_lg_en_1.0.0_3.0_1669287223603.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = nlp.T5Transformer() \ .pretrained("finsum_news_headers_lg" ,"en", "finance/models") \ .setTask("summarization") \ .setInputCols(["documents"]) \ .setMaxOutputLength(512) \ .setOutputCol("summaries") data_df = spark.createDataFrame([["FTX is expected to make its debut appearance Tuesday in Delaware bankruptcy court, where its new management is expected to recount events leading up to the cryptocurrency platform’s sudden collapse and explain the steps it has since taken to secure customer funds and other assets."]]).toDF("text") pipeline = nlp.Pipeline().setStages([document_assembler, t5]) results = pipeline.fit(data_df).transform(data_df) results.select("summaries.result").show(truncate=False) ```

## Results ```bash FTX to Make Debut in Delaware Bankruptcy Court Tuesday. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finsum_news_headers_lg| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[summaries]| |Language:|en| |Size:|925.6 MB| ## References In-house JSL financial summarized news. --- layout: model title: Sentence Entity Resolver for NDC (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_ndc date: 2022-04-18 tags: [ndc, entity_resolution, licensed, en, clinical] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.3.2 spark_version: 2.4 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps clinical entities and concepts (like drugs/ingredients) to [National Drug Codes](https://www.fda.gov/drugs/drug-approvals-and-databases/national-drug-code-directory) using `sbiobert_base_cased_mli` Sentence Bert Embeddings. It also returns package options and alternative drugs in the all_k_aux_label column. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_NDC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_NDC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_ndc_en_3.3.2_2.4_1650298194939.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_ndc_en_3.3.2_2.4_1650298194939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = SentenceDetectorDLModel.pretrained()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") posology_ner = MedicalNerModel.pretrained("ner_posology_greedy", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk")\ .setWhiteList(["DRUG"]) c2doc = Chunk2Doc()\ .setInputCols("ner_chunk")\ .setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sentence_embeddings") ndc_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_ndc", "en", "clinical/models") \ .setInputCols(["ner_chunk", "sentence_embeddings"]) \ .setOutputCol("ndc")\ .setDistanceFunction("EUCLIDEAN")\ .setCaseSensitive(False) resolver_pipeline = Pipeline(stages = [ documentAssembler, sentenceDetector, tokenizer, word_embeddings, posology_ner, ner_converter, c2doc, sbert_embedder, ndc_resolver ]) data = spark.createDataFrame([["""The patient was given aspirin 81 mg and metformin 500 mg"""]]).toDF("text") result = resolver_pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val clinical_ner = MedicalNerModel.pretrained("ner_posology_greedy", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") .setWhiteList(Array("DRUG")) val c2doc = new Chunk2Doc() .setInputCols("ner_chunk") .setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sentence_embeddings") val ndc_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_ndc", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sentence_embeddings")) .setOutputCol("ndc") .setDistanceFunction("EUCLIDEAN") .setCaseSensitive(False) val resolver_pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, word_embeddings, posology_ner, ner_converter, c2doc, sbert_embedder, ndc_resolver )) val clinical_note = Seq("The patient was given aspirin 81 mg and metformin 500 mg").toDS.toDF("text") val results = resolver_pipeline.fit(clinical_note).transform(clinical_note) ``` {:.nlu-block} ```python import nlu nlu.load("en.resolve.ndc").predict("""The patient was given aspirin 81 mg and metformin 500 mg""") ```

## Results ```bash +----------------+----------+----------------------------------------------------------------------------------------------------+ | ner_chunk| ndc_code| aux_labels| +----------------+----------+----------------------------------------------------------------------------------------------------+ | aspirin 81 mg|41250-0780|{'packages': "['1 BOTTLE, PLASTIC in 1 PACKAGE (41250-780-01) > 120 TABLET, DELAYED RELEASE in 1...| |metformin 500 mg|62207-0491|{'packages': "['5000 TABLET in 1 POUCH (62207-491-31)', '25000 TABLET in 1 CARTON (62207-491-35)'...| +----------------+----------+----------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_ndc| |Compatibility:|Healthcare NLP 3.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[ndc]| |Language:|en| |Size:|932.2 MB| |Case sensitive:|false| ## References It is trained on U.S. FDA 2022-NDC Codes dataset. --- layout: model title: Legal Security Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_security_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, security, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_security_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `security-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `security-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_security_agreement_en_1.0.0_3.0_1668111327495.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_security_agreement_en_1.0.0_3.0_1668111327495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[security-agreement]| |[other]| |[other]| |[security-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_security_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.92 0.96 0.94 85 security-agreement 0.92 0.82 0.87 40 accuracy - - 0.92 125 macro-avg 0.92 0.89 0.91 125 weighted-avg 0.92 0.92 0.92 125 ``` --- layout: model title: Fast Neural Machine Translation Model from English to Sino-Tibetan Languages author: John Snow Labs name: opus_mt_en_sit date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, sit, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `sit` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_sit_xx_2.7.0_2.4_1609169720133.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_sit_xx_2.7.0_2.4_1609169720133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_sit| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering (from thatdramebaazguy) author: John Snow Labs name: roberta_qa_movie_roberta_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `movie-roberta-squad` is a English model originally trained by `thatdramebaazguy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_movie_roberta_squad_en_4.0.0_3.0_1655729079202.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_movie_roberta_squad_en_4.0.0_3.0_1655729079202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_movie_roberta_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_movie_roberta_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.movie_squad.roberta.by_thatdramebaazguy").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_movie_roberta_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|466.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/thatdramebaazguy/movie-roberta-squad --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson TFWav2Vec2ForCTC from jessiejohnson author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson` is a English model originally trained by jessiejohnson. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson_en_4.2.0_3.0_1664019369584.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson_en_4.2.0_3.0_1664019369584.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab_by_jessiejohnson| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|228.0 MB| --- layout: model title: Toxic Comment Classification author: John Snow Labs name: multiclassifierdl_use_toxic date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [en, open_source, text_classification] supported: true annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Discussing things you care about can be difficult. The threat of abuse and harassment online means that many people stop expressing themselves and give up on seeking different opinions. Platforms struggle to effectively facilitate conversations, leading many communities to limit or completely shut down user comments. The Conversation AI team, a research initiative founded by Jigsaw and Google (both a part of Alphabet) is working on tools to help improve the online conversation. One area of focus is the study of negative online behaviors, like toxic comments (i.e. comments that are rude, disrespectful, or otherwise likely to make someone leave a discussion). So far they’ve built a range of publicly available models served through the Perspective API, including toxicity. But the current models still make errors, and they don’t allow users to select which types of toxicity they’re interested in finding (e.g. some platforms may be fine with profanity, but not with other types of toxic content). Automatically detect identity hate, insult, obscene, severe toxic, threat, or toxic content in SM comments using our out-of-the-box Spark NLP Multiclassifier DL. ## Predicted Entities `toxic`, `severe_toxic`, `identity_hate`, `insult`, `obscene`, `threat` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_MULTILABEL_TOXIC/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_MULTILABEL_TOXIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/multiclassifierdl_use_toxic_en_2.7.1_2.4_1611231604648.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/multiclassifierdl_use_toxic_en_2.7.1_2.4_1611231604648.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") use = UniversalSentenceEncoder.pretrained() \ .setInputCols(["document"])\ .setOutputCol("use_embeddings") docClassifier = MultiClassifierDLModel.pretrained("multiclassifierdl_use_toxic") \ .setInputCols(["use_embeddings"])\ .setOutputCol("category")\ .setThreshold(0.5) pipeline = Pipeline( stages = [ document, use, docClassifier ]) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") .setCleanupMode("shrink") val use = UniversalSentenceEncoder.pretrained() .setInputCols("document") .setOutputCol("use_embeddings") val docClassifier = MultiClassifierDLModel.pretrained("multiclassifierdl_use_toxic") .setInputCols("use_embeddings") .setOutputCol("category") .setThreshold(0.5f) val pipeline = new Pipeline() .setStages( Array( documentAssembler, use, docClassifier ) ) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.toxic").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|multiclassifierdl_use_toxic| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[use_embeddings]| |Output Labels:|[category]| |Language:|en| ## Data Source https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/overview ## Benchmarking ```bash precision recall f1-score support 0 0.56 0.30 0.39 127 1 0.71 0.70 0.70 761 2 0.76 0.72 0.74 824 3 0.55 0.21 0.31 147 4 0.79 0.38 0.51 50 5 0.94 1.00 0.97 1504 micro avg 0.83 0.80 0.81 3413 macro avg 0.72 0.55 0.60 3413 weighted avg 0.81 0.80 0.80 3413 samples avg 0.84 0.83 0.80 3413 F1 micro averaging: 0.8113432835820896 ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-32-finetuned-squad-seed-4` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1657192524287.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4_en_4.0.0_3.0_1657192524287.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_32_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|377.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-32-finetuned-squad-seed-4 --- layout: model title: Gujarati RoBERTa Embeddings (from surajp) author: John Snow Labs name: roberta_embeddings_RoBERTa_hindi_guj_san date: 2022-04-14 tags: [roberta, embeddings, gu, open_source] task: Embeddings language: gu edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `RoBERTa-hindi-guj-san` is a Gujarati model orginally trained by `surajp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_RoBERTa_hindi_guj_san_gu_3.4.2_3.0_1649948207679.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_RoBERTa_hindi_guj_san_gu_3.4.2_3.0_1649948207679.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_RoBERTa_hindi_guj_san| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|gu| |Size:|252.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/surajp/RoBERTa-hindi-guj-san - https://github.com/goru001/inltk - https://www.kaggle.com/disisbig/hindi-wikipedia-articles-172k - https://www.kaggle.com/disisbig/gujarati-wikipedia-articles - https://www.kaggle.com/disisbig/sanskrit-wikipedia-articles - https://twitter.com/parmarsuraj99 - https://www.linkedin.com/in/parmarsuraj99/ --- layout: model title: English RobertaForQuestionAnswering (from phiyodr) author: John Snow Labs name: roberta_qa_roberta_large_finetuned_squad2 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-finetuned-squad2` is a English model originally trained by `phiyodr`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_finetuned_squad2_en_4.0.0_3.0_1655737049100.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_finetuned_squad2_en_4.0.0_3.0_1655737049100.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_finetuned_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.large.by_phiyodr").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_finetuned_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/phiyodr/roberta-large-finetuned-squad2 - https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json - https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/ - https://arxiv.org/abs/1907.11692 - https://arxiv.org/abs/1806.03822 - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Pipeline to Detect Living Species (w2v_cc_300d) author: John Snow Labs name: ner_living_species_pipeline date: 2023-03-13 tags: [it, ner, clinical, licensed] task: Named Entity Recognition language: it edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species](https://nlp.johnsnowlabs.com/2022/06/23/ner_living_species_it_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_it_4.3.0_3.2_1678703906819.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_it_4.3.0_3.2_1678703906819.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_pipeline", "it", "clinical/models") text = '''Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_pipeline", "it", "clinical/models") val text = "Una donna di 74 anni è stata ricoverata con dolore addominale diffuso, ipossia e astenia di 2 settimane di evoluzione. La sua storia personale includeva ipertensione in trattamento con amiloride/idroclorotiazide e dislipidemia controllata con lovastatina. La sua storia familiare era: madre morta di cancro gastrico, fratello con cirrosi epatica di eziologia sconosciuta e sorella con carcinoma epatocellulare. Lo studio eziologico delle diverse cause di malattia epatica cronica comprendeva: virus epatotropi (HBV, HCV) e HIV, studio dell'autoimmunità, ceruloplasmina, ferritina e porfirine nelle urine, tutti risultati negativi. Il paziente è stato messo in trattamento anticoagulante con acenocumarolo e diuretici a tempo indeterminato." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:------------|-------------:| | 0 | donna | 4 | 8 | HUMAN | 0.9992 | | 1 | personale | 133 | 141 | HUMAN | 0.9982 | | 2 | madre | 285 | 289 | HUMAN | 0.9979 | | 3 | fratello | 317 | 324 | HUMAN | 0.9857 | | 4 | sorella | 373 | 379 | HUMAN | 0.9847 | | 5 | virus epatotropi | 493 | 508 | SPECIES | 0.7906 | | 6 | HBV | 511 | 513 | SPECIES | 0.9833 | | 7 | HCV | 516 | 518 | SPECIES | 0.991 | | 8 | HIV | 523 | 525 | SPECIES | 0.991 | | 9 | paziente | 634 | 641 | HUMAN | 0.9978 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|it| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Pipeline to Detect Pathogen, Medical Condition and Medicine author: John Snow Labs name: ner_pathogen_pipeline date: 2022-06-29 tags: [licensed, clinical, en, pathogen, ner, medicine, medical_condition, pipeline] task: Pipeline Healthcare language: en nav_key: models edition: Healthcare NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_pathogen](https://nlp.johnsnowlabs.com/2022/06/28/ner_pathogen_en_3_0.html) model. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_pathogen_pipeline_en_4.0.0_3.0_1656527387514.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_pathogen_pipeline_en_4.0.0_3.0_1656527387514.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_pathogen_pipeline", "en", "clinical/models") result = pipeline.fullAnnotate("""Racecadotril is an antisecretory medication and it has better tolerability than loperamide. Diarrhea is the condition of having loose, liquid or watery bowel movements each day. Signs of dehydration often begin with loss of the normal stretchiness of the skin. This can progress to loss of skin color, a fast heart rate as it becomes more severe. While it has been speculated that rabies virus, Lyssavirus and Ephemerovirus could be transmitted through aerosols, studies have concluded that this is only feasible in limited conditions.""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_pathogen_pipeline", "en", "clinical/models") val result = pipeline.fullAnnotate("""Racecadotril is an antisecretory medication and it has better tolerability than loperamide. Diarrhea is the condition of having loose, liquid or watery bowel movements each day. Signs of dehydration often begin with loss of the normal stretchiness of the skin. This can progress to loss of skin color, a fast heart rate as it becomes more severe. While it has been speculated that rabies virus, Lyssavirus and Ephemerovirus could be transmitted through aerosols, studies have concluded that this is only feasible in limited conditions.""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.pathogen.pipeline").predict("""Racecadotril is an antisecretory medication and it has better tolerability than loperamide. Diarrhea is the condition of having loose, liquid or watery bowel movements each day. Signs of dehydration often begin with loss of the normal stretchiness of the skin. This can progress to loss of skin color, a fast heart rate as it becomes more severe. While it has been speculated that rabies virus, Lyssavirus and Ephemerovirus could be transmitted through aerosols, studies have concluded that this is only feasible in limited conditions.""") ```

## Results ```bash |chunk |ner_label | +---------------+----------------+ |Racecadotril |Medicine | |loperamide |Medicine | |Diarrhea |MedicalCondition| |dehydration |MedicalCondition| |skin color |MedicalCondition| |fast heart rate|MedicalCondition| |rabies virus |Pathogen | |Lyssavirus |Pathogen | |Ephemerovirus |Pathogen | +---------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_pathogen_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: French CamemBert Embeddings (from cmarkea) author: John Snow Labs name: camembert_embeddings_distilcamembert_base date: 2022-05-23 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilcamembert-base` is a French model orginally trained by `cmarkea`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_distilcamembert_base_fr_3.4.4_3.0_1653321109186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_distilcamembert_base_fr_3.4.4_3.0_1653321109186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_distilcamembert_base| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|256.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/cmarkea/distilcamembert-base - https://arxiv.org/abs/1910.01108 --- layout: model title: Legal NER for NDA (Non-solicitation Clauses) author: John Snow Labs name: legner_nda_non_solicitation date: 2023-04-10 tags: [en, legal, licensed, ner, nda, non_solicitation] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `NON_SOLIC` clause with a proper classifier (use `legmulticlf_mnda_sections_paragraph_other` model for that purpose). It will extract the following entities: `NON_SOLIC_ACTION`, `NON_SOLIC_OBJECT`, `NON_SOLIC_IND_OBJECT`, and `NON_SOLIC_SUBJECT`. ## Predicted Entities `NON_SOLIC_ACTION`, `NON_SOLIC_OBJECT`, `NON_SOLIC_IND_OBJECT`, `NON_SOLIC_SUBJECT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_non_solicitation_en_1.0.0_3.0_1681096605002.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_non_solicitation_en_1.0.0_3.0_1681096605002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_non_solicitation", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""During the 12-month period commencing on the date of this Agreement, each Party agrees that it will not permit any of its officers, directors, or employees or any direct or indirect subsidiary or any employment agency retained by a Party or any such subsidiary, in each case who is or becomes aware of the negotiation of a possible Transaction between the Parties, to solicit for employment with such Party or with any of its direct or indirect subsidiaries any employee of the other Party or any of its direct or indirect subsidiaries;"""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +----------+--------------------+ |chunk |ner_label | +----------+--------------------+ |Party |NON_SOLIC_SUBJECT | |officers |NON_SOLIC_IND_OBJECT| |directors |NON_SOLIC_IND_OBJECT| |employees |NON_SOLIC_IND_OBJECT| |agency |NON_SOLIC_SUBJECT | |solicit |NON_SOLIC_ACTION | |employment|NON_SOLIC_OBJECT | |employee |NON_SOLIC_IND_OBJECT| +----------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_non_solicitation| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support NON_SOLIC_ACTION 1.00 0.95 0.98 22 NON_SOLIC_IND_OBJECT 0.67 0.90 0.76 29 NON_SOLIC_OBJECT 0.81 0.94 0.87 18 NON_SOLIC_SUBJECT 0.86 0.86 0.86 21 micro-avg 0.80 0.91 0.85 90 macro-avg 0.83 0.91 0.87 90 weighted-avg 0.82 0.91 0.86 90 ``` --- layout: model title: English asr_wav2vec2_large_robust_swbd_300h TFWav2Vec2ForCTC from facebook author: John Snow Labs name: pipeline_asr_wav2vec2_large_robust_swbd_300h date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_robust_swbd_300h` is a English model originally trained by facebook. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_robust_swbd_300h_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_swbd_300h_en_4.2.0_3.0_1664041796149.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_robust_swbd_300h_en_4.2.0_3.0_1664041796149.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_robust_swbd_300h', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_robust_swbd_300h", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_robust_swbd_300h| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|757.5 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Extract Test Entities from Voice of the Patient Documents (embeddings_clinical) author: John Snow Labs name: ner_vop_test date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, test] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts test mentions from the documents transferred from the patient’s own sentences. ## Predicted Entities `VitalTest`, `Test`, `Measurements`, `TestResult` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_en_4.4.3_3.0_1686076606665.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_test_en_4.4.3_3.0_1686076606665.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------|:------------| | thyroid levels | Test | | blood test | Test | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_test| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 VitalTest 159 24 13 172 0.87 0.92 0.90 Test 1040 101 168 1208 0.91 0.86 0.89 Measurements 139 14 47 186 0.91 0.75 0.82 TestResult 336 71 188 524 0.83 0.64 0.72 macro_avg 1674 210 416 2090 0.88 0.79 0.83 micro_avg 1674 210 416 2090 0.89 0.80 0.84 ``` --- layout: model title: Context Spell Checker Pipeline for English author: ahmedlone127 name: spellcheck_dl_pipeline date: 2022-06-14 tags: [spellcheck, spell, spellcheck_pipeline, spelling_corrector, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained spellchecker pipeline is built on the top of [spellcheck_dl](https://nlp.johnsnowlabs.com/2022/04/02/spellcheck_dl_en_2_4.html) model. This pipeline is for PySpark 2.4.x users with SparkNLP 3.4.2 and above. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/spellcheck_dl_pipeline_en_4.0.0_3.0_1655213660333.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/spellcheck_dl_pipeline_en_4.0.0_3.0_1655213660333.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash [{'checked': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'weather', '.'], 'document': ['During the summer we have the best ueather.'], 'token': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'ueather', '.']}, {'checked': ['I', 'have', 'a', 'black', 'leather', 'jacket', ',', 'so', 'nice', '.'], 'document': ['I have a black ueather jacket, so nice.'], 'token': ['I', 'have', 'a', 'black', 'ueather', 'jacket', ',', 'so', 'nice', '.']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_dl_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|99.8 MB| ## Included Models - DocumentAssembler - TokenizerModel - ContextSpellCheckerModel --- layout: model title: Universal Sentence Encoder XLING English and Spanish author: John Snow Labs name: tfhub_use_xling_en_es date: 2020-12-08 task: Embeddings language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 deprecated: true tags: [embeddings, open_source, xx] supported: true annotator: UniversalSentenceEncoder article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The Universal Sentence Encoder Cross-lingual (XLING) module is an extension of the Universal Sentence Encoder that includes training on multiple tasks across languages. The multi-task training setup is based on the paper "Learning Cross-lingual Sentence Representations via a Multi-task Dual Encoder". This specific module is trained on English and Spanish (en-es) tasks, and optimized for greater-than-word length text, such as sentences, phrases or short paragraphs. It is trained on a variety of data sources and tasks, with the goal of learning text representations that are useful out-of-the-box for a number of applications. The input to the module is variable length English or Spanish text and the output is a 512 dimensional vector. We note that one does not need to specify the language that the input is in, as the model was trained such that English and Spanish text with similar meanings will have similar (high dot product score) embeddings. We also note that this model can be used for monolingual English (and potentially monolingual Spanish) tasks with comparable or even better performance than the purely English Universal Sentence Encoder. Note: This model only works on Linux and macOS operating systems and is not compatible with Windows due to the incompatibility of the SentencePiece library. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/tfhub_use_xling_en_es_xx_2.7.0_2.4_1607440558771.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/tfhub_use_xling_en_es_xx_2.7.0_2.4_1607440558771.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_xling_en_es", "xx") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP', 'Me encanta usar SparkNLP']], ["text"])) ``` ```scala ... val embeddings = UniversalSentenceEncoder.pretrained("tfhub_use_xling_en_es", "xx") .setInputCols("document") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I love NLP", "Me encanta usar SparkNLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP", "Me encanta usar SparkNLP"] embeddings_df = nlu.load('xx.use.xling_en_es').predict(text, output_level='sentence') embeddings_df ```

## Results It gives a 512-dimensional vector of the sentences. ```bash xx_use_xling_en_es_embeddings sentence 0 [-0.02727784588932991, 0.022969702258706093, 0... I love NLP 1 [-0.01980777457356453, 0.03035994991660118, 0.... Me encanta usar SparkNLP ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|tfhub_use_xling_en_es| |Compatibility:|Spark NLP 2.7.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|xx| ## Data Source This embeddings model is imported from [https://tfhub.dev/google/universal-sentence-encoder-xling/en-es/1](https://tfhub.dev/google/universal-sentence-encoder-xling/en-es/1) --- layout: model title: Extract Clinical Entities from Voice of the Patient Documents (embeddings_clinical) author: John Snow Labs name: ner_vop date: 2023-06-06 tags: [licensed, clinical, en, ner, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts healthcare-related terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `Gender`, `Employment`, `Age`, `BodyPart`, `Substance`, `Form`, `PsychologicalCondition`, `Vaccine`, `Drug`, `DateTime`, `ClinicalDept`, `Laterality`, `Test`, `AdmissionDischarge`, `Disease`, `VitalTest`, `Dosage`, `Duration`, `RelationshipStatus`, `Route`, `Allergen`, `Frequency`, `Symptom`, `Procedure`, `HealthStatus`, `InjuryOrPoisoning`, `Modifier`, `Treatment`, `SubstanceQuantity`, `MedicalDevice`, `TestResult` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_en_4.4.3_3.0_1686072829593.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_en_4.4.3_3.0_1686072829593.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[indemnity-agreement]| |[other]| |[other]| |[indemnity-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indemnity_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support indemnity-agreement 0.96 0.94 0.95 50 other 0.97 0.98 0.98 111 accuracy - - 0.97 161 macro-avg 0.97 0.96 0.96 161 weighted-avg 0.97 0.97 0.97 161 ``` --- layout: model title: Arabic Bert Embeddings (Medium) author: John Snow Labs name: bert_embeddings_bert_medium_arabic date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-medium-arabic` is a Arabic model orginally trained by `asafaya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_medium_arabic_ar_3.4.2_3.0_1649678487166.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_medium_arabic_ar_3.4.2_3.0_1649678487166.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_medium_arabic| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|158.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/asafaya/bert-medium-arabic - https://traces1.inria.fr/oscar/ - http://commoncrawl.org/ - https://dumps.wikimedia.org/backup-index.html - https://github.com/google-research/bert - https://www.tensorflow.org/tfrc - https://github.com/alisafaya/Arabic-BERT --- layout: model title: Extract Clinical Problem Entities from Voice of the Patient Documents (embeddings_clinical) author: John Snow Labs name: ner_vop_problem date: 2023-06-06 tags: [licensed, clinical, ner, en, problem, vop] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts clinical problems from the documents transferred from the patient’s own sentences using a granular taxonomy. ## Predicted Entities `PsychologicalCondition`, `Disease`, `Symptom`, `HealthStatus`, `Modifier`, `InjuryOrPoisoning` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_en_4.4.3_3.0_1686075641877.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_problem_en_4.4.3_3.0_1686075641877.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:---------------------|:------------| | pain | Symptom | | fatigue | Symptom | | rheumatoid arthritis | Disease | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_problem| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 PsychologicalCondition 413 40 31 444 0.91 0.93 0.92 Disease 1748 288 267 2015 0.86 0.87 0.86 Symptom 3726 668 849 4575 0.85 0.81 0.83 HealthStatus 81 23 26 107 0.78 0.76 0.77 Modifier 845 259 294 1139 0.77 0.74 0.75 InjuryOrPoisoning 117 25 59 176 0.82 0.66 0.74 macro_avg 6930 1303 1526 8456 0.83 0.80 0.81 micro_avg 6930 1303 1526 8456 0.84 0.82 0.83 ``` --- layout: model title: Depression Classifier (PHS-BERT) for Tweets author: John Snow Labs name: bert_sequence_classifier_depression_twitter date: 2022-08-09 tags: [public_health, en, licensed, sequence_classification, mental_health, depression, twitter] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [PHS-BERT](https://arxiv.org/abs/2204.04521) based tweet classification model that can classify whether tweets contain depressive text. ## Predicted Entities `depression`, `no-depression` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_depression_twitter_en_4.0.2_3.0_1660051816827.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_depression_twitter_en_4.0.2_3.0_1660051816827.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python # Sample Python Code document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") tokenizer = Tokenizer() \ .setInputCols(["document"]) \ .setOutputCol("token") sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_depression_twitter", "en", "clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) data = spark.createDataFrame([ ["Do what makes you happy, be with who makes you smile, laugh as much as you breathe, and love as long as you live!"], ["Everything is a lie, everyone is fake, I'm so tired of living"] ]).toDF("text") result = pipeline.fit(data).transform(data) result.select("text", "class.result").show(truncate=False) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_depression_twitter", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documenter, tokenizer, sequenceClassifier)) val data = Seq(Array( "Do what makes you happy, be with who makes you smile, laugh as much as you breathe, and love as long as you live!", "Everything is a lie, everyone is fake, I'm so tired of living" )).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_sequence.depression_twitter").predict("""Do what makes you happy, be with who makes you smile, laugh as much as you breathe, and love as long as you live!""") ```

## Results ```bash +-----------------------------------------------------------------------------------------------------------------+---------------+ |text |result | +-----------------------------------------------------------------------------------------------------------------+---------------+ |Do what makes you happy, be with who makes you smile, laugh as much as you breathe, and love as long as you live!|[no-depression]| |Everything is a lie, everyone is fake, I'm so tired of living |[depression] | +-----------------------------------------------------------------------------------------------------------------+---------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_depression_twitter| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References Curated from several academic and in-house datasets. ## Benchmarking ```bash label precision recall f1-score support minimum 0.97 0.98 0.97 1411 high-depression 0.95 0.92 0.93 595 accuracy - - 0.96 2006 macro-avg 0.96 0.95 0.95 2006 weighted-avg 0.96 0.96 0.96 2006 ``` --- layout: model title: Pipeline to Detect Temporal Relations for Clinical Events (Enriched) author: John Snow Labs name: re_temporal_events_enriched_clinical_pipeline date: 2022-03-31 tags: [licensed, clinical, relation_extraction, event, enriched, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_temporal_events_enriched_clinical](https://nlp.johnsnowlabs.com/2020/09/28/re_temporal_events_enriched_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_temporal_events_enriched_clinical_pipeline_en_3.4.1_3.0_1648734605627.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_temporal_events_enriched_clinical_pipeline_en_3.4.1_3.0_1648734605627.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ | | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | +====+============+===========+=================+===============+===============================================+============+=================+===============+==========================+==============+ | 0 | OVERLAP | PROBLEM | 54 | 98 | longstanding intermittent right low back pain | OCCURRENCE | 121 | 144 | a motor vehicle accident | 0.532308 | +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ | 1 | AFTER | DATE | 171 | 179 | that time | PROBLEM | 201 | 219 | any specific injury | 0.577288 | +----+------------+-----------+-----------------+---------------+-----------------------------------------------+------------+-----------------+---------------+--------------------------+--------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_temporal_events_enriched_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: English RobertaForQuestionAnswering Cased model (from AyushPJ) author: John Snow Labs name: roberta_qa_ai_club_inductions_21_nlp date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ai-club-inductions-21-nlp-roBERTa` is a English model originally trained by `AyushPJ`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_en_4.3.0_3.0_1674208962596.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_en_4.3.0_3.0_1674208962596.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ai_club_inductions_21_nlp","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ai_club_inductions_21_nlp","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ai_club_inductions_21_nlp| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|465.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AyushPJ/ai-club-inductions-21-nlp-roBERTa --- layout: model title: Portuguese Named Entity Recognition (from m-lin20) author: John Snow Labs name: roberta_ner_satellite_instrument_roberta_NER date: 2022-05-03 tags: [roberta, ner, open_source, pt] task: Named Entity Recognition language: pt edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `satellite-instrument-roberta-NER` is a Portuguese model orginally trained by `m-lin20`. ## Predicted Entities `instrument`, `satellite` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_satellite_instrument_roberta_NER_pt_3.4.2_3.0_1651594391122.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_satellite_instrument_roberta_NER_pt_3.4.2_3.0_1651594391122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_satellite_instrument_roberta_NER| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|pt| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m-lin20/satellite-instrument-roberta-NER - https://github.com/Tsinghua-mLin/satellite-instrument-NER --- layout: model title: Legal Indemnification NER (Light, md) author: John Snow Labs name: legner_indemnifications_md date: 2022-12-01 tags: [en, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_indemnification_clause` Text Classifier to select only these paragraphs; This is a Legal Named Entity Recognition Model to identify the Subject (who), Action (web), Object(the indemnification) and Indirect Object (to whom) from Indemnification clauses. This is a `md` (medium version) of the classifier, trained with more data and being more resistent to false positives outside the specific section, which may help to run it at whole document level (although not recommended). ## Predicted Entities `INDEMNIFICATION`, `INDEMNIFICATION_SUBJECT`, `INDEMNIFICATION_ACTION`, `INDEMNIFICATION_INDIRECT_OBJECT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_indemnifications_md_en_1.0.0_3.0_1669894326703.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_indemnifications_md_en_1.0.0_3.0_1669894326703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------+---------------------------------+ | token| ner_label| +----------+---------------------------------+ | The| O| | Company| O| | shall| B-INDEMNIFICATION_ACTION| | protect| I-INDEMNIFICATION_ACTION| | and| O| | indemnify| B-INDEMNIFICATION_ACTION| | the| O| | Supplier|B-INDEMNIFICATION_INDIRECT_OBJECT| | against| O| | any| O| | damages| B-INDEMNIFICATION| | ,| O| | losses| B-INDEMNIFICATION| | or| O| | costs| B-INDEMNIFICATION| |whatsoever| O| +----------+---------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_indemnifications_md| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References In-house annotated examples from CUAD legal dataset ## Benchmarking ```bash label tp fp fn prec rec f1 I-INDEMNIFICATION_ACTION 9 2 0 0.8181818 1.0 0.90000004 B-INDEMNIFICATION_INDIRECT_OBJECT 24 7 0 0.7741935 1.0 0.8727273 B-INDEMNIFICATION_SUBJECT 5 2 0 0.71428573 1.0 0.8333334 I-INDEMNIFICATION_SUBJECT 3 0 0 1.0 1.0 1.0 B-INDEMNIFICATION 23 2 0 0.92 1.0 0.9583333 I-INDEMNIFICATION_INDIRECT_OBJECT 9 3 2 0.75 0.8181818 0.78260875 B-INDEMNIFICATION_ACTION 9 4 0 0.6923077 1.0 0.8181818 I-INDEMNIFICATION 5 5 0 0.5 1.0 0.6666667 Macro-average 87 25 2 0.77112114 0.97727275 0.8620434 Micro-average 87 25 2 0.77678573 0.9775281 0.86567163 ``` --- layout: model title: German ElectraForQuestionAnswering model (from deutsche-telekom) author: John Snow Labs name: electra_qa_base_squad2 date: 2022-06-22 tags: [de, open_source, electra, question_answering] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-de-squad2` is a German model originally trained by `deutsche-telekom`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_squad2_de_4.0.0_3.0_1655920425196.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_squad2_de_4.0.0_3.0_1655920425196.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_squad2","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Was ist mein Name?", "Mein Name ist Clara und ich lebe in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_squad2","de") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Was ist mein Name?", "Mein Name ist Clara und ich lebe in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.squadv2.electra.base").predict("""Was ist mein Name?|||"Mein Name ist Clara und ich lebe in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|de| |Size:|414.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/deutsche-telekom/electra-base-de-squad2 --- layout: model title: Word2Vec Embeddings in Piedmontese (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, pms, open_source] task: Embeddings language: pms edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pms_3.4.1_3.0_1647451242344.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_pms_3.4.1_3.0_1647451242344.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_unqover_roberta_base_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_unqover_roberta_base_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base.by_tli8hf").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_unqover_roberta_base_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|463.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/tli8hf/unqover-roberta-base-squad --- layout: model title: Pipeline to Classify Texts into TREC-6 Classes author: ahmedlone127 name: bert_sequence_classifier_trec_coarse_pipeline date: 2022-06-14 tags: [bert_sequence, trec, coarse, bert, en, open_source] task: Text Classification language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: false article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_sequence_classifier_trec_coarse_en](https://nlp.johnsnowlabs.com/2021/11/06/bert_sequence_classifier_trec_coarse_en.html). The TREC dataset for question classification consists of open-domain, fact-based questions divided into broad semantic categories. You can check the official documentation of the dataset, entities, etc. [here](https://search.r-project.org/CRAN/refmans/textdata/html/dataset_trec.html). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/community.johnsnowlabs.com/ahmedlone127/bert_sequence_classifier_trec_coarse_pipeline_en_4.0.0_3.0_1655211903928.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://community.johnsnowlabs.com/ahmedlone127/bert_sequence_classifier_trec_coarse_pipeline_en_4.0.0_3.0_1655211903928.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash ['LOC'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_trec_coarse_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Community| |Language:|en| |Size:|406.6 MB| ## Included Models - DocumentAssembler - TokenizerModel - BertForSequenceClassification --- layout: model title: English image_classifier_vit_gtsrb_model ViTForImageClassification from bazyl author: John Snow Labs name: image_classifier_vit_gtsrb_model date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_gtsrb_model` is a English model originally trained by bazyl. ## Predicted Entities `Children crossing`, `Double curve`, `Road work`, `Yield`, `Beware of ice/snow`, `Speed limit (70km/h)`, `Bicycles crossing`, `Roundabout mandatory`, `Speed limit (30km/h)`, `Keep left`, `Dangerous curve left`, `No vehicles`, `End of no passing`, `Bumpy road`, `Speed limit (50km/h)`, `Turn left ahead`, `Speed limit (20km/h)`, `General caution`, `Speed limit (100km/h)`, `End speed + passing limits`, `Go straight or right`, `Dangerous curve right`, `Speed limit (80km/h)`, `Slippery road`, `Turn right ahead`, `No passing veh over 3.5 tons`, `Speed limit (60km/h)`, `Pedestrians`, `Right-of-way at intersection`, `Priority road`, `End of speed limit (80km/h)`, `Road narrows on the right`, `No entry`, `Stop`, `Wild animals crossing`, `Veh > 3.5 tons prohibited`, `End no passing veh > 3.5 tons`, `Go straight or left`, `Speed limit (120km/h)`, `Ahead only`, `Keep right`, `Traffic signals`, `No passing` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_gtsrb_model_en_4.1.0_3.0_1660166650134.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_gtsrb_model_en_4.1.0_3.0_1660166650134.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_gtsrb_model", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_gtsrb_model", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_gtsrb_model| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|322.0 MB| --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Tianle) author: John Snow Labs name: distilbert_qa_tianle_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Tianle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_tianle_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769458679.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_tianle_base_uncased_finetuned_squad_en_4.3.0_3.0_1672769458679.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tianle_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_tianle_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_tianle_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Tianle/distilbert-base-uncased-finetuned-squad --- layout: model title: Word2Vec Embeddings in Egyptian Arabic (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, arz, open_source] task: Embeddings language: arz edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_arz_3.4.1_3.0_1647294515164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_arz_3.4.1_3.0_1647294515164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|w2v_cc_300d| |Type:|embeddings| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[embeddings]| |Language:|arz| |Size:|205.5 MB| |Case sensitive:|false| |Dimension:|300| --- layout: model title: Legal Participation Agreement Document Classifier (Longformer) author: John Snow Labs name: legclf_participation_agreement date: 2022-11-10 tags: [en, legal, classification, agreement, participation, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_participation_agreement` model is a Legal Longformer Document Classifier to classify if the document belongs to the class `participation-agreement` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If not, let us know and we can carry out another approach for you: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. But this theoretically should not be required. ## Predicted Entities `participation-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_participation_agreement_en_1.0.0_3.0_1668110456181.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_participation_agreement_en_1.0.0_3.0_1668110456181.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ | result| +-------+ |[participation-agreement]| |[other]| |[other]| |[participation-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_participation_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.7 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.99 0.98 85 participation-agreement 0.97 0.90 0.93 31 accuracy - - 0.97 116 macro-avg 0.97 0.95 0.96 116 weighted-avg 0.97 0.97 0.97 116 ``` --- layout: model title: Stopwords Remover for Finnish language (822 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, fi, open_source] task: Stop Words Removal language: fi edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_fi_3.4.1_3.0_1646672293430.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_fi_3.4.1_3.0_1646672293430.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[] | +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|fi| |Size:|3.7 KB| --- layout: model title: Fast Neural Machine Translation Model from English to Tetun Dili author: John Snow Labs name: opus_mt_en_tdt date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, tdt, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `tdt` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_tdt_xx_2.7.0_2.4_1609168336715.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_tdt_xx_2.7.0_2.4_1609168336715.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_tdt| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Kannada RoBERTa Embeddings (from Naveen-k) author: John Snow Labs name: roberta_embeddings_KanBERTo date: 2022-04-14 tags: [roberta, embeddings, kn, open_source] task: Embeddings language: kn edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `KanBERTo` is a Kannada model orginally trained by `Naveen-k`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_KanBERTo_kn_3.4.2_3.0_1649948277756.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_KanBERTo_kn_3.4.2_3.0_1649948277756.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_KanBERTo| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|kn| |Size:|314.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/Naveen-k/KanBERTo - https://en.wikipedia.org/wiki/Kannada - https://traces1.inria.fr/oscar/files/compressed-orig/kn.txt.gz - https://traces1.inria.fr/oscar/ --- layout: model title: English BertForQuestionAnswering model (from ArpanZS) author: John Snow Labs name: bert_qa_debug_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `debug_squad` is a English model orginally trained by `ArpanZS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_debug_squad_en_4.0.0_3.0_1654187393032.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_debug_squad_en_4.0.0_3.0_1654187393032.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_debug_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_debug_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_ArpanZS").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_debug_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|408.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ArpanZS/debug_squad --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_bert_medium_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-medium-finetuned-squad` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_finetuned_squad_en_4.0.0_3.0_1654183705887.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_finetuned_squad_en_4.0.0_3.0_1654183705887.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_medium_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_medium_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.medium.by_anas-awadalla").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_medium_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|154.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/bert-medium-finetuned-squad --- layout: model title: Spanish RobertaForSequenceClassification Base Cased model (from mrm8488) author: John Snow Labs name: roberta_sequence_classifier_ruperta_base_finetuned_pawsx date: 2022-07-13 tags: [es, open_source, roberta, sequence_classification] task: Text Classification language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForSequenceClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `RuPERTa-base-finetuned-pawsx-es` is a Spanish model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_ruperta_base_finetuned_pawsx_es_4.0.0_3.0_1657716331993.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_sequence_classifier_ruperta_base_finetuned_pawsx_es_4.0.0_3.0_1657716331993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") classifier = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_ruperta_base_finetuned_pawsx","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("class") pipeline = Pipeline(stages=[documentAssembler, tokenizer, classifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val classifer = RoBertaForSequenceClassification.pretrained("roberta_sequence_classifier_ruperta_base_finetuned_pawsx","es") .setInputCols(Array("document", "token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, classifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_sequence_classifier_ruperta_base_finetuned_pawsx| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|472.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-pawsx-es --- layout: model title: French RoBERTa Embeddings (from abhilash1910) author: John Snow Labs name: roberta_embeddings_french_roberta date: 2022-04-14 tags: [roberta, embeddings, fr, open_source] task: Embeddings language: fr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `french-roberta` is a French model orginally trained by `abhilash1910`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_french_roberta_fr_3.4.2_3.0_1649947893823.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_french_roberta_fr_3.4.2_3.0_1649947893823.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_french_roberta| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fr| |Size:|255.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/abhilash1910/french-roberta --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_finetuned_squad_r3f date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad-r3f` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_squad_r3f_en_4.3.0_3.0_1674217712089.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_finetuned_squad_r3f_en_4.3.0_3.0_1674217712089.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_squad_r3f","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_finetuned_squad_r3f","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_finetuned_squad_r3f| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-finetuned-squad-r3f --- layout: model title: Legal Organizations Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_organizations_bert date: 2023-03-05 tags: [en, legal, classification, clauses, organizations, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Organizations` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Organizations`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_organizations_bert_en_1.0.0_3.0_1678050638474.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_organizations_bert_en_1.0.0_3.0_1678050638474.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_organizations_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Organizations]| |[Other]| |[Other]| |[Organizations]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_organizations_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Organizations 0.98 0.94 0.96 47 Other 0.96 0.99 0.97 70 accuracy - - 0.97 117 macro-avg 0.97 0.96 0.96 117 weighted-avg 0.97 0.97 0.97 117 ``` --- layout: model title: English BertForQuestionAnswering model (from Sounak) author: John Snow Labs name: bert_qa_bert_large_finetuned date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-finetuned` is a English model orginally trained by `Sounak`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_finetuned_en_4.0.0_3.0_1654536299509.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_large_finetuned_en_4.0.0_3.0_1654536299509.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_large_finetuned","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_large_finetuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.large.by_Sounak").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_large_finetuned| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Sounak/bert-large-finetuned --- layout: model title: Multilingual DistilBertForQuestionAnswering Base Cased model (from ruselkomp) author: John Snow Labs name: distilbert_qa_ruselkomp_base_cased_finetuned_squad date: 2023-01-03 tags: [xx, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: xx edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multilingual-cased-finetuned-squad` is a Multilingual model originally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_ruselkomp_base_cased_finetuned_squad_xx_4.3.0_3.0_1672767256602.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_ruselkomp_base_cased_finetuned_squad_xx_4.3.0_3.0_1672767256602.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ruselkomp_base_cased_finetuned_squad","xx")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_ruselkomp_base_cased_finetuned_squad","xx") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_ruselkomp_base_cased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|xx| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/distilbert-base-multilingual-cased-finetuned-squad --- layout: model title: XLM-RoBERTa Base, CoNLL-03 NER Pipeline author: John Snow Labs name: xlm_roberta_base_token_classifier_conll03_pipeline date: 2022-04-21 tags: [open_source, ner, token_classifier, xlm_roberta, conll03, xlm, base, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [xlm_roberta_base_token_classifier_conll03](https://nlp.johnsnowlabs.com/2021/10/03/xlm_roberta_base_token_classifier_conll03_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650542851685.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_base_token_classifier_conll03_pipeline_en_3.4.1_3.0_1650542851685.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("classifierdl_bert_sentiment_pipeline", lang = "es") result1 = pipeline.annotate("Estoy seguro de que esta vez pasará la entrevista.") result2 = pipeline.annotate("Soy una persona que intenta desayunar todas las mañanas sin falta.") result3 = pipeline.annotate("No estoy seguro de si mi salario mensual es suficiente para vivir.") ``` ```scala val pipeline = new PretrainedPipeline("classifierdl_bert_sentiment_pipeline", lang = "es") val result1 = pipeline.annotate("Estoy seguro de que esta vez pasará la entrevista.")(0) val result2 = pipeline.annotate("Soy una persona que intenta desayunar todas las mañanas sin falta.")(0) val result3 = pipeline.annotate("No estoy seguro de si mi salario mensual es suficiente para vivir.")(0) ```

## Results ```bash ['POSITIVE'] ['NEUTRAL'] ['NEGATIVE'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_sentiment_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.3.0+| |License:|Open Source| |Edition:|Official| |Language:|es| ## Included Models - DocumentAssembler - BertSentenceEmbeddings - ClassifierDLModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from kevinbror) author: John Snow Labs name: distilbert_qa_kevinbror_base_uncased_finetuned_squad2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad2` is a English model originally trained by `kevinbror`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_kevinbror_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773532721.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_kevinbror_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773532721.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kevinbror_base_uncased_finetuned_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_kevinbror_base_uncased_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_kevinbror_base_uncased_finetuned_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/kevinbror/distilbert-base-uncased-finetuned-squad2 --- layout: model title: Translate English to Ukrainian Pipeline author: John Snow Labs name: translate_en_uk date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, uk, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `uk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_uk_xx_2.7.0_2.4_1609686168178.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_uk_xx_2.7.0_2.4_1609686168178.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_uk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_uk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.uk').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_uk| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Afrikaans to Russian author: John Snow Labs name: opus_mt_af_ru date: 2021-06-01 tags: [open_source, seq2seq, translation, af, ru, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: af target languages: ru {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_af_ru_xx_3.1.0_2.4_1622551937327.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_af_ru_xx_3.1.0_2.4_1622551937327.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_af_ru| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Living Species (w2v_cc_300d) author: John Snow Labs name: ner_living_species date: 2022-06-23 tags: [gl, ner, clinical, licensed] task: Named Entity Recognition language: gl edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in Galician which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `w2v_cc_300d` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_gl_3.5.3_3.0_1655976346794.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_gl_3.5.3_3.0_1655976346794.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d", "gl")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species", "gl", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Muller de 45 anos, sen antecedentes médicos de interese, que foi remitida á consulta de dermatoloxía de urxencias por lesións faciales de tres semanas de evolución. A paciente non presentaba lesións noutras localizaciones nin outra clínica de interese. No seu centro de saúde prescribíronlle corticoides tópicos ante a sospeita de picaduras de artrópodos e unha semana despois, antivirales orais baixo o diagnóstico de posible infección herpética. As lesións interferían de forma notable na súa vida persoal e profesional xa que traballaba de face ao púbico. Unha semana máis tarde o diagnóstico foi confirmado ao resultar o cultivo positivo a Staphylococcus aureus."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","gl") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species", "gl","clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Muller de 45 anos, sen antecedentes médicos de interese, que foi remitida á consulta de dermatoloxía de urxencias por lesións faciales de tres semanas de evolución. A paciente non presentaba lesións noutras localizaciones nin outra clínica de interese. No seu centro de saúde prescribíronlle corticoides tópicos ante a sospeita de picaduras de artrópodos e unha semana despois, antivirales orais baixo o diagnóstico de posible infección herpética. As lesións interferían de forma notable na súa vida persoal e profesional xa que traballaba de face ao púbico. Unha semana máis tarde o diagnóstico foi confirmado ao resultar o cultivo positivo a Staphylococcus aureus.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("gl.med_ner.living_species").predict("""Muller de 45 anos, sen antecedentes médicos de interese, que foi remitida á consulta de dermatoloxía de urxencias por lesións faciales de tres semanas de evolución. A paciente non presentaba lesións noutras localizaciones nin outra clínica de interese. No seu centro de saúde prescribíronlle corticoides tópicos ante a sospeita de picaduras de artrópodos e unha semana despois, antivirales orais baixo o diagnóstico de posible infección herpética. As lesións interferían de forma notable na súa vida persoal e profesional xa que traballaba de face ao púbico. Unha semana máis tarde o diagnóstico foi confirmado ao resultar o cultivo positivo a Staphylococcus aureus.""") ```

## Results ```bash +---------------------+-------+ |ner_chunk |label | +---------------------+-------+ |Muller |HUMAN | |paciente |HUMAN | |artrópodos |SPECIES| |antivirales |SPECIES| |herpética |SPECIES| |púbico |HUMAN | |Staphylococcus aureus|SPECIES| +---------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|gl| |Size:|15.2 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.88 0.97 0.92 2952 B-SPECIES 0.54 0.91 0.67 3333 I-HUMAN 0.74 0.75 0.74 206 I-SPECIES 0.59 0.85 0.70 1297 micro-avg 0.65 0.92 0.76 7788 macro-avg 0.69 0.87 0.76 7788 weighted-avg 0.68 0.92 0.77 7788 ``` --- layout: model title: Translate English to Kuanyama Pipeline author: John Snow Labs name: translate_en_kj date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, kj, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `kj` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_kj_xx_2.7.0_2.4_1609687416916.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_kj_xx_2.7.0_2.4_1609687416916.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_kj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_kj", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.kj').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_kj| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Detect Person, Location, Organization, and Miscellaneous entities in Arabic (ANERcorp) author: John Snow Labs name: aner_cc_300d date: 2022-08-09 tags: [ner, ar, open_source] task: Named Entity Recognition language: ar edition: Spark NLP 4.0.2 spark_version: 3.0 supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses Arabic word embeddings to find 4 different types of entities in Arabic text. It is trained using `arabic_w2v_cc_300d` word embeddings, so please use the same embeddings in the pipeline. ## Predicted Entities `PER`, `LOC`, `ORG`, `MISC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/aner_cc_300d_ar_4.0.2_3.0_1660030385202.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/aner_cc_300d_ar_4.0.2_3.0_1660030385202.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("aner_cc_300d", "ar") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز") ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = new SentenceDetector() .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = WordEmbeddingsModel.pretrained("arabic_w2v_cc_300d", "ar",) .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("aner_cc_300d", "ar") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val nlp_pipeline = new Pipeline().setStages(Array( documentAssembler, sentenceDetector, tokenizer, embeddings, ner, ner_converter)) val data = Seq("""في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز""").toDS.toDF("text") val result = nlp_pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ar.ner").predict("""في عام 1918 حررت قوات الثورة العربية دمشق بمساعدة من الإنكليز""") ```

## Results ```bash | | ner_chunk | entity | |---:|-------------------------:|-------------:| | 0 | قوات الثورة العربية | ORG | | 1 | دمشق | LOC | | 2 | الإنكليز | PER | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|aner_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 4.0.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|ar| |Size:|14.8 MB| ## References This model is trained on data obtained from [http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp](http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp) ## Benchmarking ```bash label tp fp fn prec rec f1 B-LOC 163 28 34 0.853403 0.827411 0.840206 I-ORG 60 10 5 0.857142 0.923077 0.888889 I-MIS 124 53 53 0.700565 0.700565 0.700565 I-LOC 64 20 23 0.761904 0.735632 0.748538 B-MIS 297 71 52 0.807065 0.851003 0.828452 I-PER 84 23 13 0.785046 0.865979 0.823530 B-ORG 54 9 12 0.857142 0.818181 0.837210 B-PER 182 26 33 0.875 0.846512 0.860520 Macro-average 1028 240 225 0.812159 0.821045 0.816578 Micro-average 1028 240 225 0.810726 0.820431 0.815550 ``` --- layout: model title: Part of Speech for Latvian author: John Snow Labs name: pos_ud_lvtb date: 2021-03-09 tags: [part_of_speech, open_source, latvian, pos_ud_lvtb, lv] task: Part of Speech Tagging language: lv edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - PROPN - VERB - ADJ - PUNCT - PRON - PART - CCONJ - NOUN - AUX - ADP - ADV - DET - NUM - SCONJ - SYM - X - INTJ {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_lvtb_lv_3.0.0_3.0_1615292214572.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_lvtb_lv_3.0.0_3.0_1615292214572.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_lvtb", "lv") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Sveiki no John Sniega Labs! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_lvtb", "lv") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Sveiki no John Sniega Labs! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Sveiki no John Sniega Labs! ""] token_df = nlu.load('lv.pos').predict(text) token_df ```

## Results ```bash token pos 0 Sveiki VERB 1 no ADP 2 John PROPN 3 Sniega PROPN 4 Labs PROPN 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_lvtb| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|lv| --- layout: model title: Legal Third Supplemental Indenture Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_third_supplemental_indenture_bert date: 2023-02-02 tags: [en, legal, classification, third, supplemental, indenture, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_third_supplemental_indenture_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `third-supplemental-indenture` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `third-supplemental-indenture`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_third_supplemental_indenture_bert_en_1.0.0_3.0_1675360680501.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_third_supplemental_indenture_bert_en_1.0.0_3.0_1675360680501.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_third_supplemental_indenture_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[third-supplemental-indenture]| |[other]| |[other]| |[third-supplemental-indenture]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_third_supplemental_indenture_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.96 0.92 0.94 122 third-supplemental-indenture 0.81 0.90 0.85 49 accuracy - - 0.91 171 macro-avg 0.89 0.91 0.90 171 weighted-avg 0.92 0.91 0.91 171 ``` --- layout: model title: Pipeline to Detect Cellular/Molecular Biology Entities (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_cellular_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, berfortokenclassification, cellular, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_cellular](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_cellular_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_pipeline_en_3.4.1_3.0_1647889939388.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_cellular_pipeline_en_3.4.1_3.0_1647889939388.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") pipeline.annotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_ner_cellular_pipeline", "en", "clinical/models") pipeline.annotate("Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.cellular_pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash +-----------------------------------------------------------+---------+ |chunk |ner_label| +-----------------------------------------------------------+---------+ |intracellular signaling proteins |protein | |human T-cell leukemia virus type 1 promoter |DNA | |Tax |protein | |Tax-responsive element 1 |DNA | |cyclic AMP-responsive members |protein | |CREB/ATF family |protein | |transcription factors |protein | |Tax |protein | |human T-cell leukemia virus type 1 Tax-responsive element 1|DNA | |TRE-1 |DNA | |lacZ gene |DNA | |CYC1 promoter |DNA | |TRE-1 |DNA | |cyclic AMP response element-binding protein |protein | |CREB |protein | |CREB |protein | |GAL4 activation domain |protein | |GAD |protein | |reporter gene |DNA | |Tax |protein | +-----------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_cellular_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverter --- layout: model title: Part of Speech for Tamil (pos_ttb) author: John Snow Labs name: pos_ttb date: 2021-03-10 tags: [open_source, pos, ta] supported: true task: Part of Speech Tagging language: ta edition: Spark NLP 2.7.5 spark_version: 2.4 annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron` architecture. ## Predicted Entities - ADJ - ADP - ADV - AUX - CCONJ - DET - NOUN - NUM - PART - PRON - PROPN - PUNCT - VERB - X {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ttb_ta_2.7.5_2.4_1615399578187.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ttb_ta_2.7.5_2.4_1615399578187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols("sentence")\ .setOutputCol("token") pos = PerceptronModel.pretrained("pos_ttb", "ta") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, posTagger ]) example = spark.createDataFrame([['எனவே ஐநா சபை மூலமாக நிதி உதவியை அளிக்கும் ஆறு இந்தியாவுக்கு தகவல் அனுப்பிய் உள்ளோம் என அந் நாட்டின் வெளியுறவுத் துறை செய்தித்தொடர்பாளர் அப்துல் பாசித் தெரிவித்தார் . ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = Tokenizer() .setInputCols("sentence") .setOutputCol("token") val pos = PerceptronModel.pretrained("pos_ttb", "ta") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector,tokenizer, pos)) val data = Seq("எனவே ஐநா சபை மூலமாக நிதி உதவியை அளிக்கும் ஆறு இந்தியாவுக்கு தகவல் அனுப்பிய் உள்ளோம் என அந் நாட்டின் வெளியுறவுத் துறை செய்தித்தொடர்பாளர் அப்துல் பாசித் தெரிவித்தார் . ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""எனவே ஐநா சபை மூலமாக நிதி உதவியை அளிக்கும் ஆறு இந்தியாவுக்கு தகவல் அனுப்பிய் உள்ளோம் என அந் நாட்டின் வெளியுறவுத் துறை செய்தித்தொடர்பாளர் அப்துல் பாசித் தெரிவித்தார் . ""] token_df = nlu.load('ta.pos.ttb').predict(text) token_df ```

## Results ```bash +---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------+ |text |result | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------+ |எனவே ஐநா சபை மூலமாக நிதி உதவியை அளிக்கும் ஆறு இந்தியாவுக்கு தகவல் அனுப்பிய் உள்ளோம் என அந் நாட்டின் வெளியுறவுத் துறை செய்தித்தொடர்பாளர் அப்துல் பாசித் தெரிவித்தார் .|[ADV, PROPN, NOUN, ADP, NOUN, NOUN, ADJ, PART, PROPN, NOUN, VERB, AUX, PART, DET, NOUN, NOUN, NOUN, NOUN, PROPN, PROPN, VERB, PUNCT]| +---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ttb| |Compatibility:|Spark NLP 2.7.5+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[pos]| |Language:|ta| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) data set. ## Benchmarking ```bash | | precision | recall | f1-score | support | |--------------|-----------|--------|----------|---------| | ADJ | 0.21 | 0.58 | 0.31 | 53 | | ADP | 0.76 | 0.41 | 0.53 | 68 | | ADV | 0.70 | 0.57 | 0.63 | 75 | | AUX | 0.73 | 0.74 | 0.73 | 151 | | CCONJ | 1.00 | 1.00 | 1.00 | 8 | | DET | 0.80 | 0.69 | 0.74 | 29 | | NOUN | 0.72 | 0.80 | 0.76 | 526 | | NUM | 0.87 | 0.71 | 0.78 | 91 | | PART | 0.82 | 0.81 | 0.82 | 168 | | PRON | 0.73 | 0.75 | 0.74 | 61 | | PROPN | 0.67 | 0.57 | 0.62 | 249 | | PUNCT | 0.83 | 0.89 | 0.86 | 190 | | VERB | 0.65 | 0.51 | 0.57 | 319 | | X | 0.00 | 0.00 | 0.00 | 1 | | accuracy | | | 0.70 | 1989 | | macro avg | 0.68 | 0.65 | 0.65 | 1989 | | weighted avg | 0.72 | 0.70 | 0.70 | 1989 | ``` --- layout: model title: Detect Clinical Entities in Romanian (Bert, Base, Cased) author: John Snow Labs name: ner_clinical_bert date: 2022-08-12 tags: [licensed, clinical, ro, ner, bert] task: Named Entity Recognition language: ro edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true recommended: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract clinical entities from Romanian clinical texts. This model is trained using `bert_base_cased` embeddings. ## Predicted Entities `Measurements`, `Form`, `Symptom`, `Route`, `Procedure`, `Disease_Syndrome_Disorder`, `Score`, `Drug_Ingredient`, `Pulse`, `Frequency`, `Date`, `Body_Part`, `Drug_Brand_Name`, `Time`, `Direction`, `Medical_Device`, `Imaging_Technique`, `Test`, `Imaging_Findings`, `Imaging_Test`, `Test_Result`, `Weight`, `Clinical_Dept`, `Units` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_ro_4.0.2_3.0_1660295520992.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_clinical_bert_ro_4.0.2_3.0_1660295520992.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +--------------------------+-------------------------+ |chunks |entities | +--------------------------+-------------------------+ |Angio CT cardio-toracic |Imaging_Test | |Atrezie |Disease_Syndrome_Disorder| |valva pulmonara |Body_Part | |Hipoplazie |Disease_Syndrome_Disorder| |VS |Body_Part | |Atrezie |Disease_Syndrome_Disorder| |VAV stang |Body_Part | |Anastomoza Glenn |Disease_Syndrome_Disorder| |Tromboza |Disease_Syndrome_Disorder| |Sectia Clinica Cardiologie|Clinical_Dept | |GE Revolution HD |Medical_Device | |Branula albastra |Medical_Device | |membrului superior drept |Body_Part | |Scout |Body_Part | |30 ml |Dosage | |Iomeron 350 |Drug_Ingredient | |2.2 ml/s |Dosage | |20 ml |Dosage | |ser fiziologic |Drug_Ingredient | |angio-CT |Imaging_Test | +--------------------------+-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_clinical_bert| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.2 MB| ## Benchmarking ```bash label precision recall f1-score support Body_Part 0.91 0.93 0.92 679 Clinical_Dept 0.68 0.65 0.67 97 Date 0.99 0.99 0.99 87 Direction 0.66 0.76 0.70 50 Disease_Syndrome_Disorder 0.73 0.76 0.74 121 Dosage 0.78 1.00 0.87 38 Drug_Ingredient 0.90 0.94 0.92 48 Form 1.00 1.00 1.00 6 Imaging_Findings 0.86 0.82 0.84 201 Imaging_Technique 0.92 0.92 0.92 26 Imaging_Test 0.93 0.98 0.95 205 Measurements 0.71 0.69 0.70 214 Medical_Device 0.85 0.81 0.83 42 Pulse 0.82 1.00 0.90 9 Route 1.00 0.91 0.95 33 Score 1.00 0.98 0.99 41 Time 1.00 1.00 1.00 28 Units 0.60 0.93 0.73 88 Weight 0.82 1.00 0.90 9 micro-avg 0.84 0.87 0.86 2037 macro-avg 0.70 0.74 0.72 2037 weighted-avg 0.84 0.87 0.85 2037 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab2_by_ahmad573 TFWav2Vec2ForCTC from ahmad573 author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab2_by_ahmad573 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab2_by_ahmad573` is a English model originally trained by ahmad573. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab2_by_ahmad573_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_ahmad573_en_4.2.0_3.0_1664036825737.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab2_by_ahmad573_en_4.2.0_3.0_1664036825737.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_ahmad573", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab2_by_ahmad573", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab2_by_ahmad573| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English asr_Wav2Vec2_XLSR_Bengali_10500 TFWav2Vec2ForCTC from shoubhik author: John Snow Labs name: asr_Wav2Vec2_XLSR_Bengali_10500 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_Wav2Vec2_XLSR_Bengali_10500` is a English model originally trained by shoubhik. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_Wav2Vec2_XLSR_Bengali_10500_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_XLSR_Bengali_10500_en_4.2.0_3.0_1664104957251.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_Wav2Vec2_XLSR_Bengali_10500_en_4.2.0_3.0_1664104957251.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_Wav2Vec2_XLSR_Bengali_10500", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_Wav2Vec2_XLSR_Bengali_10500", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_Wav2Vec2_XLSR_Bengali_10500| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|3.6 GB| --- layout: model title: Translate English to Pangasinan Pipeline author: John Snow Labs name: translate_en_pag date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, pag, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `pag` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_pag_xx_2.7.0_2.4_1609691698746.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_pag_xx_2.7.0_2.4_1609691698746.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_pag", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_pag", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.pag').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_pag| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Extract relations between effects of using multiple drugs (ReDL) author: John Snow Labs name: redl_drug_drug_interaction_biobert date: 2023-01-14 tags: [relation_extraction, en, licensed, clinical, tensorflow] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract potential improvements or harmful effects of Drug-Drug interactions (DDIs) when two or more drugs are taken at the same time or at a certain interval. ## Predicted Entities `DDI-advise`, `DDI-effect`, `DDI-false`, `DDI-int`, `DDI-mechanism` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_DRUG_DRUG_INT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_4.2.4_3.0_1673734887835.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_drug_drug_interaction_biobert_en_4.2.4_3.0_1673734887835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ | relation|entity1|entity1_begin|entity1_end| chunk1|entity2|entity2_begin|entity2_end| chunk2|confidence| +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ |DDI-false| DRUG| 5| 17|carbamazepine| DRUG| 62| 73|aripiprazole|0.91685396| +---------+-------+-------------+-----------+-------------+-------+-------------+-----------+------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_drug_drug_interaction_biobert| |Compatibility:|Healthcare NLP 4.2.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|401.7 MB| ## References Trained on DDI Extraction corpus. ## Benchmarking ```bash label Recall Precision F1 Support DDI-advise 0.758 0.874 0.812 211 DDI-effect 0.759 0.754 0.756 348 DDI-false 0.977 0.957 0.967 4097 DDI-int 0.175 0.458 0.253 63 DDI-mechanism 0.783 0.853 0.816 281 Avg. 0.690 0.779 0.721 - ``` --- layout: model title: Legal Definitions and interpretation Clause Binary Classifier author: John Snow Labs name: legclf_definitions_and_interpretation_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `definitions-and-interpretation` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `definitions-and-interpretation` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_definitions_and_interpretation_clause_en_1.0.0_3.2_1660122329652.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_definitions_and_interpretation_clause_en_1.0.0_3.2_1660122329652.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_definitions_and_interpretation_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[definitions-and-interpretation]| |[other]| |[other]| |[definitions-and-interpretation]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_definitions_and_interpretation_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support definitions-and-interpretation 0.96 0.94 0.95 54 other 0.98 0.99 0.98 156 accuracy - - 0.98 210 macro-avg 0.97 0.97 0.97 210 weighted-avg 0.98 0.98 0.98 210 ``` --- layout: model title: English image_classifier_vit_Tomato_Leaf_Classifier ViTForImageClassification from Aftabhussain author: John Snow Labs name: image_classifier_vit_Tomato_Leaf_Classifier date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_Tomato_Leaf_Classifier` is a English model originally trained by Aftabhussain. ## Predicted Entities `Bacterial_spot`, `Healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Tomato_Leaf_Classifier_en_4.1.0_3.0_1660167088791.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_Tomato_Leaf_Classifier_en_4.1.0_3.0_1660167088791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_Tomato_Leaf_Classifier", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_Tomato_Leaf_Classifier", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_Tomato_Leaf_Classifier| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from ManqingLiu) author: John Snow Labs name: xlmroberta_ner_manqingliu_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `ManqingLiu`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_manqingliu_base_finetuned_panx_de_4.1.0_3.0_1660429790495.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_manqingliu_base_finetuned_panx_de_4.1.0_3.0_1660429790495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_manqingliu_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_manqingliu_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_manqingliu_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ManqingLiu/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: Legal Specific Performance Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_specific_performance_bert date: 2023-03-05 tags: [en, legal, classification, clauses, specific_performance, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Specific_Performance` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Specific_Performance`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_specific_performance_bert_en_1.0.0_3.0_1678050609899.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_specific_performance_bert_en_1.0.0_3.0_1678050609899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_specific_performance_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Specific_Performance]| |[Other]| |[Other]| |[Specific_Performance]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_specific_performance_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.4 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Other 0.98 0.98 0.98 53 Specific_Performance 0.97 0.97 0.97 34 accuracy - - 0.98 87 macro-avg 0.98 0.98 0.98 87 weighted-avg 0.98 0.98 0.98 87 ``` --- layout: model title: Pipeline to Extraction of Clinical Abbreviations and Acronyms author: John Snow Labs name: ner_abbreviation_clinical_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_abbreviation_clinical](https://nlp.johnsnowlabs.com/2021/12/30/ner_abbreviation_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_abbreviation_clinical_pipeline_en_3.4.1_3.0_1647874931143.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_abbreviation_clinical_pipeline_en_3.4.1_3.0_1647874931143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_abbreviation_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Gravid with estimated fetal weight of 6-6/12 pounds. LOWER EXTREMITIES: No edema. LABORATORY DATA: Laboratory tests include a CBC which is normal. Blood Type: AB positive. Rubella: Immune. VDRL: Nonreactive. Hepatitis C surface antigen: Negative. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.") ``` ```scala val pipeline = new PretrainedPipeline("ner_abbreviation_clinical_pipeline", "en", "clinical/models") pipeline.annotate("Gravid with estimated fetal weight of 6-6/12 pounds. LOWER EXTREMITIES: No edema. LABORATORY DATA: Laboratory tests include a CBC which is normal. Blood Type: AB positive. Rubella: Immune. VDRL: Nonreactive. Hepatitis C surface antigen: Negative. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical-abbreviation.pipeline").predict("""Gravid with estimated fetal weight of 6-6/12 pounds. LOWER EXTREMITIES: No edema. LABORATORY DATA: Laboratory tests include a CBC which is normal. Blood Type: AB positive. Rubella: Immune. VDRL: Nonreactive. Hepatitis C surface antigen: Negative. HIV: Negative. One-Hour Glucose: 117. Group B strep has not been done as yet.""") ```

## Results ```bash +-----+---------+ |chunk|ner_label| +-----+---------+ |CBC |ABBR | |AB |ABBR | |VDRL |ABBR | |HIV |ABBR | +-----+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_abbreviation_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Legal Confidentiality Clause Binary Classifier author: John Snow Labs name: legclf_cuad_confidentiality_clause date: 2022-09-20 tags: [en, legal, classification, clauses, confidentiality, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `confidentiality` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `confidentiality` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_cuad_confidentiality_clause_en_1.0.0_3.2_1663693181343.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_cuad_confidentiality_clause_en_1.0.0_3.2_1663693181343.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_confidential_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[confidentiality]| |[other]| |[other]| |[confidentiality]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_cuad_confidentiality_clause| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.2 MB| ## References In-house annotations on CUAD dataset ## Benchmarking ```bash label precision recall f1-score support confidentiality 1.00 0.91 0.95 11 other 0.95 1.00 0.97 19 accuracy - - 0.97 30 macro-avg 0.97 0.95 0.96 30 weighted-avg 0.97 0.97 0.97 30 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab51 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab51 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab51` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab51_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab51_en_4.2.0_3.0_1664021491825.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab51_en_4.2.0_3.0_1664021491825.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab51", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab51", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab51| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: English image_classifier_vit_road_good_damaged_condition ViTForImageClassification from edixo author: John Snow Labs name: image_classifier_vit_road_good_damaged_condition date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_road_good_damaged_condition` is a English model originally trained by edixo. ## Predicted Entities `damaged road`, `good road` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_road_good_damaged_condition_en_4.1.0_3.0_1660171740568.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_road_good_damaged_condition_en_4.1.0_3.0_1660171740568.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_road_good_damaged_condition", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_road_good_damaged_condition", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_road_good_damaged_condition| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Spanish RoBERTa Embeddings (from MMG) author: John Snow Labs name: roberta_embeddings_mlm_spanish_roberta_base date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `mlm-spanish-roberta-base` is a Spanish model orginally trained by `MMG`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_mlm_spanish_roberta_base_es_3.4.2_3.0_1649945432879.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_mlm_spanish_roberta_base_es_3.4.2_3.0_1649945432879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_mlm_spanish_roberta_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|473.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/MMG/mlm-spanish-roberta-base - https://github.com/dccuchile/GLUES --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_768 date: 2022-12-02 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_768_zh_4.2.4_3.0_1670021751518.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_768_zh_4.2.4_3.0_1670021751518.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|224.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Icelandic DistilBertForTokenClassification Cased model (from m3hrdadfi) author: John Snow Labs name: dtilbert_token_classifier_typo_detector date: 2023-03-03 tags: [is, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: is edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `typo-detector-distilbert-is` is a Icelandic model originally trained by `m3hrdadfi`. ## Predicted Entities `TYPO` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/dtilbert_token_classifier_typo_detector_is_4.3.1_3.0_1677881909024.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/dtilbert_token_classifier_typo_detector_is_4.3.1_3.0_1677881909024.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("dtilbert_token_classifier_typo_detector","is") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("dtilbert_token_classifier_typo_detector","is") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|dtilbert_token_classifier_typo_detector| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|is| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m3hrdadfi/typo-detector-distilbert-is - https://github.com/m3hrdadfi/typo-detector/issues --- layout: model title: Spanish Named Entity Recognition (from mrm8488) author: John Snow Labs name: roberta_ner_RuPERTa_base_finetuned_ner date: 2022-05-03 tags: [roberta, ner, open_source, es] task: Named Entity Recognition language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `RuPERTa-base-finetuned-ner` is a Spanish model orginally trained by `mrm8488`. ## Predicted Entities `MISC`, `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_RuPERTa_base_finetuned_ner_es_3.4.2_3.0_1651593655495.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_RuPERTa_base_finetuned_ner_es_3.4.2_3.0_1651593655495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_RuPERTa_base_finetuned_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|470.7 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/mrm8488/RuPERTa-base-finetuned-ner - https://www.kaggle.com/nltkdata/conll-corpora - https://www.kaggle.com/nltkdata/conll-corpora - https://twitter.com/mrm8488 --- layout: model title: Detect Drug Information (Small) author: John Snow Labs name: ner_posology_small class: NerDLModel language: en nav_key: models repository: clinical/models date: 2020-04-21 task: Named Entity Recognition edition: Healthcare NLP 2.4.2 spark_version: 2.4 tags: [clinical,licensed,ner,en] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description Pretrained named entity recognition deep learning model for posology, this NER is trained with the ``embeddings_clinical`` word embeddings model, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities ``DOSAGE``, ``DRUG``, ``DURATION``, ``FORM``, ``FREQUENCY``, ``ROUTE``, ``STRENGTH``. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_POSOLOGY/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_en_2.4.2_2.4_1587513301751.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_posology_small_en_2.4.2_2.4_1587513301751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_posology_small","en","clinical/models")\ .setInputCols(["sentence","token","embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([['The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.']], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val model = NerDLModel.pretrained("ner_posology_small","en","clinical/models") .setInputCols(Array("sentence","token","embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, model, ner_converter)) val data = Seq("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.posology.small").predict("""The patient is a 30-year-old female with a long history of insulin dependent diabetes, type 2; coronary artery disease; chronic renal insufficiency; peripheral vascular disease, also secondary to diabetes; who was originally admitted to an outside hospital for what appeared to be acute paraplegia, lower extremities. She did receive a course of Bactrim for 14 days for UTI. Evidently, at some point in time, the patient was noted to develop a pressure-type wound on the sole of her left foot and left great toe. She was also noted to have a large sacral wound; this is in a similar location with her previous laminectomy, and this continues to receive daily care. The patient was transferred secondary to inability to participate in full physical and occupational therapy and continue medical management of her diabetes, the sacral decubitus, left foot pressure wound, and associated complications of diabetes. She is given Fragmin 5000 units subcutaneously daily, Xenaderm to wounds topically b.i.d., Lantus 40 units subcutaneously at bedtime, OxyContin 30 mg p.o. q.12 h., folic acid 1 mg daily, levothyroxine 0.1 mg p.o. daily, Prevacid 30 mg daily, Avandia 4 mg daily, Norvasc 10 mg daily, Lexapro 20 mg daily, aspirin 81 mg daily, Senna 2 tablets p.o. q.a.m., Neurontin 400 mg p.o. t.i.d., Percocet 5/325 mg 2 tablets q.4 h. p.r.n., magnesium citrate 1 bottle p.o. p.r.n., sliding scale coverage insulin, Wellbutrin 100 mg p.o. daily, and Bactrim DS b.i.d.""") ```

## Results ```bash +--------------+---------+ |chunk |ner | +--------------+---------+ |insulin |DRUG | |Bactrim |DRUG | |for 14 days |DURATION | |Fragmin |DRUG | |5000 units |DOSAGE | |subcutaneously|ROUTE | |daily |FREQUENCY| |Xenaderm |DRUG | |topically |ROUTE | |b.i.d., |FREQUENCY| |Lantus |DRUG | |40 units |DOSAGE | |subcutaneously|ROUTE | |at bedtime |FREQUENCY| |OxyContin |DRUG | |30 mg |STRENGTH | |p.o |ROUTE | |q.12 h |FREQUENCY| |folic acid |DRUG | |1 mg |STRENGTH | +--------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |----------------|----------------------------------| | Name: | ner_posology_small | | Type: | NerDLModel | | Compatibility: | Spark NLP 2.4.2+ | | License: | Licensed | |Edition:|Official| | |Input labels: | sentence, token, word_embeddings | |Output labels: | ner | | Language: | en | | Case sensitive: | False | | Dependencies: | embeddings_clinical | {:.h2_title} ## Data Source Trained on the 2018 i2b2 dataset (no FDA) with ``embeddings_clinical``. https://www.i2b2.org/NLP/Medication {:.h2_title} ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|--------------:|------:|------:|------:|---------:|---------:|---------:| | 0 | B-DRUG | 1408 | 62 | 99 | 0.957823 | 0.934307 | 0.945919 | | 1 | B-STRENGTH | 470 | 43 | 29 | 0.916179 | 0.941884 | 0.928854 | | 2 | I-DURATION | 123 | 22 | 8 | 0.848276 | 0.938931 | 0.891304 | | 3 | I-STRENGTH | 499 | 66 | 15 | 0.883186 | 0.970817 | 0.924931 | | 4 | I-FREQUENCY | 945 | 47 | 55 | 0.952621 | 0.945 | 0.948795 | | 5 | B-FORM | 365 | 13 | 12 | 0.965608 | 0.96817 | 0.966887 | | 6 | B-DOSAGE | 298 | 27 | 26 | 0.916923 | 0.919753 | 0.918336 | | 7 | I-DOSAGE | 348 | 29 | 22 | 0.923077 | 0.940541 | 0.931727 | | 8 | I-DRUG | 208 | 25 | 60 | 0.892704 | 0.776119 | 0.830339 | | 9 | I-ROUTE | 10 | 0 | 2 | 1 | 0.833333 | 0.909091 | | 10 | B-ROUTE | 467 | 4 | 25 | 0.991507 | 0.949187 | 0.969886 | | 11 | B-DURATION | 64 | 10 | 10 | 0.864865 | 0.864865 | 0.864865 | | 12 | B-FREQUENCY | 588 | 12 | 17 | 0.98 | 0.971901 | 0.975934 | | 13 | I-FORM | 264 | 5 | 4 | 0.981413 | 0.985075 | 0.98324 | | 14 | Macro-average | 6057 | 365 | 384 | 0.93387 | 0.924277 | 0.929049 | | 15 | Micro-average | 6057 | 365 | 384 | 0.943164 | 0.940382 | 0.941771 | ``` --- layout: model title: Translate English to Creoles and pidgins, French‑based Pipeline author: John Snow Labs name: translate_en_cpf date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, cpf, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `cpf` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_cpf_xx_2.7.0_2.4_1609691980246.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_cpf_xx_2.7.0_2.4_1609691980246.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_cpf", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_cpf", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.cpf').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_cpf| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English BertForQuestionAnswering model (from ofirzaf) author: John Snow Labs name: bert_qa_ofirzaf_bert_large_uncased_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-large-uncased-squad` is a English model orginally trained by `ofirzaf`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_ofirzaf_bert_large_uncased_squad_en_4.0.0_3.0_1654536654168.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_ofirzaf_bert_large_uncased_squad_en_4.0.0_3.0_1654536654168.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_ofirzaf_bert_large_uncased_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_ofirzaf_bert_large_uncased_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.large_uncased.by_ofirzaf").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_ofirzaf_bert_large_uncased_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ofirzaf/bert-large-uncased-squad --- layout: model title: Abkhazian asr_xls_r_ab_test_by_pablouribe TFWav2Vec2ForCTC from pablouribe author: John Snow Labs name: asr_xls_r_ab_test_by_pablouribe date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_pablouribe` is a Abkhazian model originally trained by pablouribe. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_ab_test_by_pablouribe_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_pablouribe_ab_4.2.0_3.0_1664021635884.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_pablouribe_ab_4.2.0_3.0_1664021635884.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_ab_test_by_pablouribe", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_ab_test_by_pablouribe", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_ab_test_by_pablouribe| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|445.7 KB| --- layout: model title: English asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4 TFWav2Vec2ForCTC from lilitket author: John Snow Labs name: pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4` is a English model originally trained by lilitket. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4_en_4.2.0_3.0_1664119982498.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4_en_4.2.0_3.0_1664119982498.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xls_r_300m_hyAM_batch4_lr4| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Costs Clause Binary Classifier author: John Snow Labs name: legclf_costs_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `costs` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `costs` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_costs_clause_en_1.0.0_3.2_1660122307454.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_costs_clause_en_1.0.0_3.2_1660122307454.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_costs_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[costs]| |[other]| |[other]| |[costs]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_costs_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support costs 0.89 0.85 0.87 48 other 0.95 0.96 0.95 128 accuracy - - 0.93 176 macro-avg 0.92 0.91 0.91 176 weighted-avg 0.93 0.93 0.93 176 ``` --- layout: model title: English RobertaForQuestionAnswering (from armageddon) author: John Snow Labs name: roberta_qa_roberta_large_squad2_covid_qa_deepset date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad2-covid-qa-deepset` is a English model originally trained by `armageddon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad2_covid_qa_deepset_en_4.0.0_3.0_1655737643329.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad2_covid_qa_deepset_en_4.0.0_3.0_1655737643329.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_squad2_covid_qa_deepset","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_squad2_covid_qa_deepset","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_squad2_covid_qa_deepset| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/armageddon/roberta-large-squad2-covid-qa-deepset --- layout: model title: Legal Remedies cumulative Clause Binary Classifier author: John Snow Labs name: legclf_remedies_cumulative_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `remedies-cumulative` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `remedies-cumulative` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_remedies_cumulative_clause_en_1.0.0_3.2_1660122931314.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_remedies_cumulative_clause_en_1.0.0_3.2_1660122931314.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_remedies_cumulative_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[remedies-cumulative]| |[other]| |[other]| |[remedies-cumulative]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_remedies_cumulative_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.96 0.98 114 remedies-cumulative 0.91 1.00 0.95 42 accuracy - - 0.97 156 macro-avg 0.96 0.98 0.97 156 weighted-avg 0.98 0.97 0.97 156 ``` --- layout: model title: English asr_wav2vec2_murad_with_some_data TFWav2Vec2ForCTC from MBMMurad author: John Snow Labs name: asr_wav2vec2_murad_with_some_data date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_murad_with_some_data` is a English model originally trained by MBMMurad. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_murad_with_some_data_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_murad_with_some_data_en_4.2.0_3.0_1664110531411.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_murad_with_some_data_en_4.2.0_3.0_1664110531411.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_murad_with_some_data", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_murad_with_some_data", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_murad_with_some_data| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_vasilis TFWav2Vec2ForCTC from vasilis author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_vasilis` is a Finnish model originally trained by vasilis. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_fi_4.2.0_3.0_1664024101617.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis_fi_4.2.0_3.0_1664024101617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis', lang = 'fi') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis", lang = "fi") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_53_finnish_by_vasilis| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|fi| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Indonesian T5ForConditionalGeneration Base Cased model (from cahya) author: John Snow Labs name: t5_cahya_base_indonesian_summarization_cased date: 2023-01-30 tags: [id, open_source, t5, tensorflow] task: Text Generation language: id edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-indonesian-summarization-cased` is a Indonesian model originally trained by `cahya`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_cahya_base_indonesian_summarization_cased_id_4.3.0_3.0_1675109672981.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_cahya_base_indonesian_summarization_cased_id_4.3.0_3.0_1675109672981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_cahya_base_indonesian_summarization_cased","id") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_cahya_base_indonesian_summarization_cased","id") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_cahya_base_indonesian_summarization_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|id| |Size:|926.2 MB| ## References - https://huggingface.co/cahya/t5-base-indonesian-summarization-cased --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from raisinbl) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_squad_2_512_1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad_2_512_1` is a English model originally trained by `raisinbl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad_2_512_1_en_4.3.0_3.0_1672773730195.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad_2_512_1_en_4.3.0_3.0_1672773730195.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_2_512_1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_2_512_1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_squad_2_512_1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/raisinbl/distilbert-base-uncased-finetuned-squad_2_512_1 --- layout: model title: Smaller BERT Sentence Embeddings (L-10_H-512_A-8) author: John Snow Labs name: sent_small_bert_L10_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_small_bert_L10_512_en_2.6.0_2.4_1598350765497.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_small_bert_L10_512_en_2.6.0_2.4_1598350765497.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_512", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_small_bert_L10_512", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.small_bert_L10_512').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash en_embed_sentence_small_bert_L10_512_embeddings sentence [-0.45785054564476013, 0.16585223376750946, 1.... I hate cancer [0.07550251483917236, 0.009794552810490131, 0.... Antibiotics aren't painkiller ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_small_bert_L10_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1 --- layout: model title: Slovak Lemmatizer author: John Snow Labs name: lemma date: 2020-05-05 11:16:00 +0800 task: Lemmatization language: sk edition: Spark NLP 2.5.0 spark_version: 2.4 tags: [lemmatizer, sk] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_sk_2.5.0_2.4_1588666524270.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_sk_2.5.0_2.4_1588666524270.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "sk") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny.") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "sk") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Okrem toho, že je kráľom severu, je John Snow anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny."""] lemma_df = nlu.load('sk.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Okrem', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=6, end=9, result='to', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=10, end=10, result=',', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=12, end=13, result='že', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=15, end=16, result='jesť', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.0+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|sk| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Dutch Named Entity Recognition (from ml6team) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_nl_emoji_ner date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, nl, open_source] task: Named Entity Recognition language: nl edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-nl-emoji-ner` is a Dutch model orginally trained by `ml6team`. ## Predicted Entities `😨`, `🚗`, `☕`, `😥`, `💰`, `😠`, `🍾`, `😍`, `😄`, `🤯` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_nl_emoji_ner_nl_3.4.2_3.0_1652809991427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_nl_emoji_ner_nl_3.4.2_3.0_1652809991427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_nl_emoji_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|nl| |Size:|851.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ml6team/xlm-roberta-base-nl-emoji-ner --- layout: model title: Classifier for Adverse Drug Events using Clinical Bert author: John Snow Labs name: classifierdl_ade_clinicalbert date: 2021-01-21 task: Text Classification language: en nav_key: models edition: Healthcare NLP 2.7.1 spark_version: 2.4 tags: [en, licensed, classifier, clinical] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify text/sentence in two categories: `True` : The sentence is talking about a possible ADE `False` : The sentences doesn’t have any information about an ADE. ## Predicted Entities `True`, `False` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/16.Adverse_Drug_Event_ADE_NER_and_Classifier.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_clinicalbert_en_2.7.1_2.4_1611244439637.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifierdl_ade_clinicalbert_en_2.7.1_2.4_1611244439637.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler().setInputCol("text").setOutputCol("document") tokenizer = Tokenizer().setInputCols(['document']).setOutputCol('token') embeddings = BertEmbeddings.pretrained('biobert_clinical_base_cased')\ .setInputCols(["document", 'token'])\ .setOutputCol("word_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["document", "word_embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifier = ClassifierDLModel.pretrained('classifierdl_ade_clinicalbert', 'en', 'clinical/models')\ .setInputCols(['document', 'token', 'sentence_embeddings']).setOutputCol('class') nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, embeddings, sentence_embeddings, classifier]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate(["I feel a bit drowsy & have a little blurred vision after taking an insulin", "I feel great after taking tylenol"]) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.ade.clinicalbert").predict("""I feel a bit drowsy & have a little blurred vision after taking an insulin""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_uk_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Pipeline to Detect Chemicals in Medical Text author: John Snow Labs name: bert_token_classifier_ner_bc4chemd_chemicals_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_bc4chemd_chemicals](https://nlp.johnsnowlabs.com/2022/07/25/bert_token_classifier_ner_bc4chemd_chemicals_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc4chemd_chemicals_pipeline_en_4.3.0_3.2_1679301435930.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_bc4chemd_chemicals_pipeline_en_4.3.0_3.2_1679301435930.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_bc4chemd_chemicals_pipeline", "en", "clinical/models") text = '''The main isolated compounds were triterpenes (alpha - amyrin, beta - amyrin, lupeol, betulin, betulinic acid, uvaol, erythrodiol and oleanolic acid) and phenolic acid derivatives from 4 - hydroxybenzoic acid (gallic and protocatechuic acids and isocorilagin).''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_bc4chemd_chemicals_pipeline", "en", "clinical/models") val text = "The main isolated compounds were triterpenes (alpha - amyrin, beta - amyrin, lupeol, betulin, betulinic acid, uvaol, erythrodiol and oleanolic acid) and phenolic acid derivatives from 4 - hydroxybenzoic acid (gallic and protocatechuic acids and isocorilagin)." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:--------------------------------|--------:|------:|:------------|-------------:| | 0 | triterpenes | 33 | 43 | CHEM | 0.99999 | | 1 | alpha - amyrin | 46 | 59 | CHEM | 0.999939 | | 2 | beta - amyrin | 62 | 74 | CHEM | 0.999679 | | 3 | lupeol | 77 | 82 | CHEM | 0.999968 | | 4 | betulin | 85 | 91 | CHEM | 0.999975 | | 5 | betulinic acid | 94 | 107 | CHEM | 0.999984 | | 6 | uvaol | 110 | 114 | CHEM | 0.99998 | | 7 | erythrodiol | 117 | 127 | CHEM | 0.999987 | | 8 | oleanolic acid | 133 | 146 | CHEM | 0.999984 | | 9 | phenolic acid | 153 | 165 | CHEM | 0.999985 | | 10 | 4 - hydroxybenzoic acid | 184 | 206 | CHEM | 0.999973 | | 11 | gallic and protocatechuic acids | 209 | 239 | CHEM | 0.999984 | | 12 | isocorilagin | 245 | 256 | CHEM | 0.999985 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_bc4chemd_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.7 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Word2Vec Embeddings in Asturian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-14 tags: [cc, embeddings, fastText, word2vec, ast, open_source] task: Embeddings language: ast edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ast_3.4.1_3.0_1647284350724.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_ast_3.4.1_3.0_1647284350724.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_term_loan_credit_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[term-loan-credit-agreement]| |[other]| |[other]| |[term-loan-credit-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_term_loan_credit_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.98 0.98 0.98 116 term-loan-credit-agreement 0.96 0.96 0.96 51 accuracy - - 0.98 167 macro-avg 0.97 0.97 0.97 167 weighted-avg 0.98 0.98 0.98 167 ``` --- layout: model title: English BertForQuestionAnswering model (from madlag) author: John Snow Labs name: bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-squadv1-x1.84-f88.7-d36-hybrid-filled-v1` is a English model orginally trained by `madlag`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1_en_4.0.0_3.0_1654181557143.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1_en_4.0.0_3.0_1654181557143.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased_1_block_sparse_0.13_v1.by_madlag").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_uncased_squadv1_x1.84_f88.7_d36_hybrid_filled_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|206.9 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/madlag/bert-base-uncased-squadv1-x1.84-f88.7-d36-hybrid-filled-v1 - https://rajpurkar.github.io/SQuAD-explorer - https://www.aclweb.org/anthology/N19-1423.pdf --- layout: model title: French CamemBert Embeddings (from cylee) author: John Snow Labs name: camembert_embeddings_cylee_generic_model date: 2022-05-31 tags: [fr, open_source, camembert, embeddings] task: Embeddings language: fr edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: CamemBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained CamemBert Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `dummy-model` is a French model orginally trained by `cylee`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/camembert_embeddings_cylee_generic_model_fr_3.4.4_3.0_1653987722726.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/camembert_embeddings_cylee_generic_model_fr_3.4.4_3.0_1653987722726.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|camembert_embeddings_cylee_generic_model| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|fr| |Size:|266.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/cylee/dummy-model --- layout: model title: Relation extraction between body parts and procedures author: John Snow Labs name: redl_bodypart_procedure_test_biobert date: 2021-02-04 task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 2.7.3 spark_version: 2.4 tags: [licensed, clinical, en, relation_extraction] supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like ‘Internal_organ_or_component’, ’External_body_part_or_region’ etc. and procedure and test entities. `1` : body part and test/procedure are related to each other. `0` : body part and test/procedure are not related to each other. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_2.7.3_2.4_1612447034744.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_2.7.3_2.4_1612447034744.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab0_by_hassnain', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab0_by_hassnain", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab0_by_hassnain| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from lijingxin) author: John Snow Labs name: xlmroberta_ner_lijingxin_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `lijingxin`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_lijingxin_base_finetuned_panx_all_xx_4.1.0_3.0_1660428690762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_lijingxin_base_finetuned_panx_all_xx_4.1.0_3.0_1660428690762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_lijingxin_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_lijingxin_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_lijingxin_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/lijingxin/xlm-roberta-base-finetuned-panx-all --- layout: model title: English RobertaForQuestionAnswering (from Firat) author: John Snow Labs name: roberta_qa_Firat_roberta_base_finetuned_squad date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-finetuned-squad` is a English model originally trained by `Firat`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_Firat_roberta_base_finetuned_squad_en_4.0.0_3.0_1655734262536.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_Firat_roberta_base_finetuned_squad_en_4.0.0_3.0_1655734262536.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_Firat_roberta_base_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_Firat_roberta_base_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base.by_Firat").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_Firat_roberta_base_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|464.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Firat/roberta-base-finetuned-squad --- layout: model title: Turkish ElectraForQuestionAnswering model (from enelpi) Discriminator Version-1 author: John Snow Labs name: electra_qa_base_discriminator_finetuned_squadv1 date: 2022-06-22 tags: [tr, open_source, electra, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-discriminator-finetuned_squadv1_tr` is a Turkish model originally trained by `enelpi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv1_tr_4.0.0_3.0_1655920559958.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_discriminator_finetuned_squadv1_tr_4.0.0_3.0_1655920559958.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv1","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_discriminator_finetuned_squadv1","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.squad.electra.base").predict("""Benim adım ne?|||"Benim adım Clara ve Berkeley'de yaşıyorum.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_discriminator_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|412.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/enelpi/electra-base-discriminator-finetuned_squadv1_tr --- layout: model title: Named Entity Recognition (NER) Model in Bengali (bengaliner_cc_300d) author: John Snow Labs name: bengaliner_cc_300d date: 2021-02-10 task: Named Entity Recognition language: bn edition: Spark NLP 2.7.3 spark_version: 2.4 tags: [open_source, bn, ner] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect 4 different types of entities in Indian text. ## Predicted Entities `PER`, `ORG`, `LOC`, `TIME` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_BN/){:.button.button-orange} [Open in Colab](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bengaliner_cc_300d_bn_2.7.3_2.4_1612957259511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bengaliner_cc_300d_bn_2.7.3_2.4_1612957259511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = WordEmbeddingsModel.pretrained("bengali_cc_300d", "bn") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = NerDLModel.pretrained("bengaliner_cc_300d", "bn") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... pipeline = Pipeline(stages=[document_assembler, tokenizer, embeddings, ner, ner_converter]) example = spark.createDataFrame([['১৯৪৮ সালে ইয়াজউদ্দিন আহম্মেদ মুন্সিগঞ্জ উচ্চ বিদ্যালয় থেকে মেট্রিক পাশ করেন এবং ১৯৫০ সালে মুন্সিগঞ্জ হরগঙ্গা কলেজ থেকে ইন্টারমেডিয়েট পাশ করেন']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala ... val embeddings = WordEmbeddingsModel.pretrained("bengali_cc_300d", "bn") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("bengaliner_cc_300d", "bn") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner, ner_converter)) val data = Seq("১৯৪৮ সালে ইয়াজউদ্দিন আহম্মেদ মুন্সিগঞ্জ উচ্চ বিদ্যালয় থেকে মেট্রিক পাশ করেন এবং ১৯৫০ সালে মুন্সিগঞ্জ হরগঙ্গা কলেজ থেকে ইন্টারমেডিয়েট পাশ করেন").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("bn.ner").predict("""১৯৪৮ সালে ইয়াজউদ্দিন আহম্মেদ মুন্সিগঞ্জ উচ্চ বিদ্যালয় থেকে মেট্রিক পাশ করেন এবং ১৯৫০ সালে মুন্সিগঞ্জ হরগঙ্গা কলেজ থেকে ইন্টারমেডিয়েট পাশ করেন""") ```

## Results ```bash +----------------------+-----------+ | ner_chunk | label | +----------------------+-----------+ | ১৯৪৮ সালে | TIME | | ইয়াজউদ্দিন আহম্মেদ | PER | | মুন্সিগঞ্জ উচ্চ বিদ্যালয় | ORG | | ১৯৫০ সালে | TIME | | মুন্সিগঞ্জ হরগঙ্গা কলেজ | ORG | +----------------------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bengaliner_cc_300d| |Type:|ner| |Compatibility:|Spark NLP 2.7.3+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token, word_embeddings]| |Output Labels:|[ner]| |Language:|bn| ## Data Source This models is trained on data obtained from https://ieeexplore.ieee.org/document/8944804 ## Benchmarking ```bash label tp fp fn prec rec f1 I-TIME 167 37 25 0.8186275 0.8697917 0.8434344 B-LOC 678 114 195 0.8560606 0.7766323 0.81441444 I-ORG 287 104 143 0.73401535 0.66744184 0.6991474 B-TIME 414 54 123 0.88461536 0.7709497 0.8238806 I-LOC 98 50 76 0.6621622 0.5632184 0.6086956 I-PER 805 38 55 0.9549229 0.93604654 0.94539046 B-ORG 446 108 225 0.8050541 0.6646796 0.72816324 B-PER 764 48 183 0.9408867 0.80675817 0.86867535 tp: 3659 fp: 553 fn: 1025 labels: 8 Macro-average prec: 0.8320431, rec: 0.75693977, f1: 0.79271656 Micro-average prec: 0.86870843, rec: 0.78116995, f1: 0.8226169 ``` --- layout: model title: Legal Usa patriot act Clause Binary Classifier author: John Snow Labs name: legclf_usa_patriot_act_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `usa-patriot-act` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `usa-patriot-act` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_usa_patriot_act_clause_en_1.0.0_3.2_1660123155724.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_usa_patriot_act_clause_en_1.0.0_3.2_1660123155724.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_usa_patriot_act_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[usa-patriot-act]| |[other]| |[other]| |[usa-patriot-act]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_usa_patriot_act_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.99 1.00 0.99 97 usa-patriot-act 1.00 0.96 0.98 25 accuracy - - 0.99 122 macro-avg 0.99 0.98 0.99 122 weighted-avg 0.99 0.99 0.99 122 ``` --- layout: model title: English image_classifier_vit_rust_image_classification_11 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_rust_image_classification_11 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_rust_image_classification_11` is a English model originally trained by SummerChiam. ## Predicted Entities `nonrust0`, `rust0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_11_en_4.1.0_3.0_1660171448218.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_rust_image_classification_11_en_4.1.0_3.0_1660171448218.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_rust_image_classification_11", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_rust_image_classification_11", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_rust_image_classification_11| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Legal NER (Headers / Subheaders) author: John Snow Labs name: legner_headers date: 2022-08-12 tags: [en, legal, ner, agreements, splitting, licensed] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Legal NER Model, aimed to carry out Section Splitting by using the Headers and Subheaders entities, detected in the document. Other models can be found to detect other parts of the document, as Headers/Subheaders, Signers, "Will-do", etc. ## Predicted Entities `HEADER`, `SUBHEADER` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/LEGALNER_HEADERS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_headers_en_1.0.0_3.2_1660298515978.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_headers_en_1.0.0_3.2_1660298515978.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = legal.NerModel.pretrained('legner_headers', 'en', 'legal/models')\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence","token","ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = [""" 2. Definitions. For purposes of this Agreement, the following terms have the meanings ascribed thereto in this Section 1. 2. Appointment as Reseller. 2.1 Appointment. The Company hereby [***]. Allscripts may also disclose Company's pricing information relating to its Merchant Processing Services and facilitate procurement of Merchant Processing Services on behalf of Sublicensed Customers, including, without limitation by references to such pricing information and Merchant Processing Services in Customer Agreements. 6 2.2 Customer Agreements."""] res = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +-----------+-----------+ | token| ner_label| +-----------+-----------+ | 2| B-HEADER| | .| I-HEADER| |Definitions| I-HEADER| | .| O| | For| O| | purposes| O| | of| O| | this| O| | Agreement| O| | ,| O| | the| O| | following| O| | terms| O| | have| O| | the| O| | meanings| O| | ascribed| O| | thereto| O| | in| O| | this| O| | Section| O| | 1|B-SUBHEADER| | .|I-SUBHEADER| | 2|I-SUBHEADER| | .|I-SUBHEADER| |Appointment| I-HEADER| | as| I-HEADER| | Reseller| I-HEADER| | .| O| | 2.1|B-SUBHEADER| |Appointment|I-SUBHEADER| | .| O| | The| O| | Company| O| | hereby| O| | [***]| O| | .| O| | Allscripts| O| | may| O| | also| O| | disclose| O| | Company's| O| | pricing| O| |information| O| | relating| O| | to| O| | its| O| | Merchant| O| | Processing| O| | Services| O| | and| O| | facilitate| O| |procurement| O| | of| O| | Merchant| O| | Processing| O| | Services| O| | on| O| | behalf| O| | of| O| |Sublicensed| O| | Customers| O| | ,| O| | including| O| | ,| O| | without| O| | limitation| O| | by| O| | references| O| | to| O| | such| O| | pricing| O| |information| O| | and| O| | Merchant| O| | Processing| O| | Services| O| | in| O| | Customer| O| | Agreements| O| | .| O| | 6| O| | 2.2|B-SUBHEADER| | Customer|I-SUBHEADER| | Agreements|I-SUBHEADER| | .| O| +-----------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_headers| |Type:|legal| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.3 MB| ## References Manual annotations on CUAD dataset ## Benchmarking ```bash label tp fp fn prec rec f1 I-HEADER 1486 40 25 0.97378767 0.98345464 0.9785973 B-SUBHEADER 744 16 14 0.97894734 0.98153037 0.9802372 I-SUBHEADER 2382 53 34 0.9782341 0.98592716 0.98206556 B-HEADER 415 4 12 0.9904535 0.97189695 0.9810875 Macro-average 5027 113 85 0.9803556 0.9807023 0.9805289 Micro-average 5027 113 85 0.97801554 0.98337245 0.98068666 ``` --- layout: model title: Stopwords Remover for Thai language (1006 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, th, open_source] task: Stop Words Removal language: th edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_th_3.4.1_3.0_1646673180836.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_th_3.4.1_3.0_1646673180836.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+ |result | +-------------------+ |[คุณไม่ดีไปกว่าฉัน]| +-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|th| |Size:|6.0 KB| --- layout: model title: Pipeline to Detect Living Species(w2v_cc_300d) author: John Snow Labs name: ner_living_species_pipeline date: 2023-03-13 tags: [es, ner, clinical, licensed] task: Named Entity Recognition language: es edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species](https://nlp.johnsnowlabs.com/2022/06/22/ner_living_species_es_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_es_4.3.0_3.2_1678706115739.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_pipeline_es_4.3.0_3.2_1678706115739.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_pipeline", "es", "clinical/models") text = '''Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_pipeline", "es", "clinical/models") val text = "Lactante varón de dos años. Antecedentes familiares sin interés. Antecedentes personales: Embarazo, parto y periodo neonatal normal. En seguimiento por alergia a legumbres, diagnosticado con diez meses por reacción urticarial generalizada con lentejas y garbanzos, con dieta de exclusión a legumbres desde entonces. En ésta visita la madre describe episodios de eritema en zona maxilar derecha con afectación ocular ipsilateral que se resuelve en horas tras la administración de corticoides. Le ha ocurrido en 5-6 ocasiones, en relación con la ingesta de alimentos previamente tolerados. Exploración complementaria: Cacahuete, ac(ige)19.2 Ku.arb/l. Resultados: Ante la sospecha clínica de Síndrome de Frey, se tranquiliza a los padres, explicándoles la naturaleza del cuadro y se cita para revisión anual." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:---------------|--------:|------:|:------------|-------------:| | 0 | Lactante varón | 0 | 13 | HUMAN | 0.9926 | | 1 | familiares | 41 | 50 | HUMAN | 0.9994 | | 2 | personales | 78 | 87 | HUMAN | 0.9987 | | 3 | neonatal | 116 | 123 | HUMAN | 0.9731 | | 4 | legumbres | 162 | 170 | SPECIES | 0.9978 | | 5 | lentejas | 243 | 250 | SPECIES | 0.9995 | | 6 | garbanzos | 254 | 262 | SPECIES | 0.9933 | | 7 | legumbres | 290 | 298 | SPECIES | 0.9991 | | 8 | madre | 334 | 338 | HUMAN | 0.9997 | | 9 | Cacahuete | 616 | 624 | SPECIES | 0.9998 | | 10 | padres | 728 | 733 | HUMAN | 0.9992 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|es| |Size:|1.3 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Fast Neural Machine Translation Model from English to Maltese author: John Snow Labs name: opus_mt_en_mt date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, mt, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `mt` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_mt_xx_2.7.0_2.4_1609169930393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_mt_xx_2.7.0_2.4_1609169930393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_mt| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Self Treatment Changes Classifier in Tweets (BioBERT) author: John Snow Labs name: bert_sequence_classifier_treatment_changes_sentiment_tweet date: 2022-08-04 tags: [en, clinical, licensed, public_health, classifier, sequence_classification, treatment_changes, sentiment, treatment] task: Text Classification language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: MedicalBertForSequenceClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a [BioBERT based](https://github.com/dmis-lab/biobert) classifier that can classify patients non-adherent to their treatments and their reasons on Twitter. ## Predicted Entities `negative`, `positive` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/PUBLIC_HEALTH_CHANGE_DRUG_TREATMENT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/PUBLIC_HEALTH_MB4SC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_treatment_changes_sentiment_tweet_en_4.0.2_3.0_1659637869883.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_sequence_classifier_treatment_changes_sentiment_tweet_en_4.0.2_3.0_1659637869883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol('text') \ .setOutputCol('document') tokenizer = Tokenizer() \ .setInputCols(['document']) \ .setOutputCol('token') sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_treatment_changes_sentiment_tweet", "en", "clinical/models")\ .setInputCols(["document",'token'])\ .setOutputCol("class") pipeline = Pipeline(stages=[ document_assembler, tokenizer, sequenceClassifier ]) data = spark.createDataFrame(["I love when they say things like this. I took that ambien instead of my thyroid pill.", "I am a 30 year old man who is not overweight but is still on the verge of needing a Lipitor prescription."], StringType()).toDF("text") result = pipeline.fit(data).transform(data) result.select("text", "class.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val sequenceClassifier = MedicalBertForSequenceClassification.pretrained("bert_sequence_classifier_treatment_changes_sentiment_tweet", "en", "clinical/models") .setInputCols(Array("document","token")) .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, sequenceClassifier)) val data = Seq(Array("I love when they say things like this. I took that ambien instead of my thyroid pill.", "I am a 30 year old man who is not overweight but is still on the verge of needing a Lipitor prescription.")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.bert_sequence.treatment_sentiment_tweets").predict("""I am a 30 year old man who is not overweight but is still on the verge of needing a Lipitor prescription.""") ```

## Results ```bash +---------------------------------------------------------------------------------------------------------+----------+ |text |result | +---------------------------------------------------------------------------------------------------------+----------+ |I love when they say things like this. I took that ambien instead of my thyroid pill. |[positive]| |I am a 30 year old man who is not overweight but is still on the verge of needing a Lipitor prescription.|[negative]| +---------------------------------------------------------------------------------------------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_sequence_classifier_treatment_changes_sentiment_tweet| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|406.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## Benchmarking ```bash label precision recall f1-score support negative 0.9515 0.9751 0.9632 1368 positive 0.6304 0.4603 0.5321 126 accuracy - - 0.9317 1494 macro-avg 0.7910 0.7177 0.7476 1494 weighted-avg 0.9244 0.9317 0.9268 1494 ``` --- layout: model title: Detect Entities (BERT) author: John Snow Labs name: ner_dl_bert date: 2020-09-08 task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [ner, en, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description `ner_dl_bert` is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. It was trained on the CoNLL 2003 text corpus. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. `ner_dl_bert` model is trained with `bert_based_cased` word embeddings, so be sure to use the same embeddings in the pipeline. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ner_dl_bert_en_2.6.0_2.4_1599550979101.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ner_dl_bert_en_2.6.0_2.4_1599550979101.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## Predicted Entities `PER`, `LOC`, `ORG`, `MISC` ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained(name='bert_base_cased', lang='en') \ .setInputCols(['document', 'token']) \ .setOutputCol('embeddings') ner_model = NerDLModel.pretrained("ner_dl_bert", "en") \ .setInputCols(["document", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([['']]).toDF('text')) result = pipeline_model.transform(spark.createDataFrame([["William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."]], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained(name="bert_base_cased", lang="en") .setInputCols(Array('document', 'token')) .setOutputCol('embeddings') val ner_model = NerDLModel.pretrained("ner_dl_bert", "en") .setInputCols(Array("document", "token", "embeddings")) .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000. He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella."""] ner_df = nlu.load('en.ner.dl.bert').predict(text, output_level = "chunk") ner_df[["entities", "entities_confidence"]] ```

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PER | |American |MISC | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_dl_bert| |Type:|ner| |Compatibility:| Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from[CoNLL 2003 Data Set](https://github.com/synalp/NER/tree/master/corpus/CoNLL-2003) --- layout: model title: Legal Romanian NER (RONEC dataset) author: John Snow Labs name: legner_ronec date: 2022-11-30 tags: [ro, ner, legal, ronec, licensed] task: Named Entity Recognition language: ro edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legner_ronec` is a Named Entity Recognition model trained on RONEC (ROmanian Named Entity Corpus). Unlike the original dataset, it has been trained with the following classes: - PERSON - proper nouns or pronouns if they refer to a person - LOC - location or geo political entity - ORG - organization - LANGUAGE - language - NAT_REL_POL - national, religious or political organizations - DATETIME - a time and date in any format, including references to time (e.g. 'yesterday') - MONEY - a monetary value, numeric or otherwise - NUMERIC - a simple numeric value, represented as digits or words - ORDINAL - an ordinal value like 'first', 'third', etc. - WORK_OF_ART - a work of art like a named TV show, painting, etc. - EVENT - a named recognizable or periodic major event ## Predicted Entities `DATETIME`, `EVENT`, `LANGUAGE`, `LOC`, `MONEY`, `NAT_REL_POL`, `NUMERIC`, `ORDINAL`, `ORG`, `PERSON`, `WORK_OF_ART` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_ronec_ro_1.0.0_3.0_1669842840646.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_ronec_ro_1.0.0_3.0_1669842840646.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_base_cased", "ro")\ .setInputCols("sentence", "token")\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_ronec", "ro", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""Guvernul de stânga italian, condus de premierul Romano Prodi, a devenit după numirea a încă trei secretari de stat, cel mai numeros Executiv din istoria Republicii italiene, având 102 membri."""]]).toDF("text") result = model.transform(data) ```

## Results ```bash +----------------------+-----------+ |ner_chunk |label | +----------------------+-----------+ |Guvernul |ORG | |italian |NAT_REL_POL| |premierul Romano Prodi|PERSON | |trei |NUMERIC | |secretari |PERSON | |Republicii italiene |LOC | |102 |NUMERIC | |membri |PERSON | +----------------------+-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_ronec| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.2 MB| ## References Dataset is available [here](https://github.com/dumitrescustefan/ronec). ## Benchmarking ```bash label precision recall f1-score support DATETIME 0.90 0.90 0.90 1070 EVENT 0.53 0.68 0.59 116 LANGUAGE 0.98 0.95 0.97 44 LOC 0.91 0.90 0.91 1699 MONEY 0.97 0.97 0.97 130 NAT_REL_POL 0.92 0.94 0.93 510 NUMERIC 0.95 0.95 0.95 970 ORDINAL 0.88 0.93 0.90 183 ORG 0.81 0.83 0.82 779 PERSON 0.89 0.91 0.90 2635 WORK_OF_ART 0.73 0.57 0.64 140 micro-avg 0.89 0.90 0.89 8276 macro-avg 0.86 0.87 0.86 8276 weighted-avg 0.89 0.90 0.89 8276 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from sunitha) author: John Snow Labs name: roberta_qa_cv_merge_ds date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `CV_Merge_DS` is a English model originally trained by `sunitha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_cv_merge_ds_en_4.3.0_3.0_1674207962257.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_cv_merge_ds_en_4.3.0_3.0_1674207962257.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cv_merge_ds","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_cv_merge_ds","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_cv_merge_ds| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/sunitha/CV_Merge_DS --- layout: model title: Word2Vec Embeddings in Lithuanian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, lt, open_source] task: Embeddings language: lt edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_lt_3.4.1_3.0_1647443135863.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_lt_3.4.1_3.0_1647443135863.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_BC5CDR_Chem_Modified_SciBERT_512| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|410.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ghadeermobasher/BC5CDR-Chem-Modified-SciBERT-512 --- layout: model title: English BertForQuestionAnswering Cased model (from songhee) author: John Snow Labs name: bert_qa_i_manual_m date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `i-manual-mbert` is a English model originally trained by `songhee`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_i_manual_m_en_4.0.0_3.0_1657189488729.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_i_manual_m_en_4.0.0_3.0_1657189488729.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_i_manual_m","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_i_manual_m","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_i_manual_m| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|666.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/songhee/i-manual-mbert --- layout: model title: English BertForQuestionAnswering model (from ericRosello) author: John Snow Labs name: bert_qa_results date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `results` is a English model orginally trained by `ericRosello`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_results_en_4.0.0_3.0_1654189231590.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_results_en_4.0.0_3.0_1654189231590.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_results","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_results","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.bert.by_ericRosello").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_results| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ericRosello/results --- layout: model title: English DistilBertForQuestionAnswering Uncased model (from SauravMaheshkar) author: John Snow Labs name: distilbert_qa_base_uncased_distilled_chaii date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-distilled-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_distilled_chaii_en_4.0.0_3.0_1654723758525.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_distilled_chaii_en_4.0.0_3.0_1654723758525.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_distilled_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_distilled_chaii","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_distilled_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/distilbert-base-uncased-distilled-chaii --- layout: model title: Extract Demographic Entities from Voice of the Patient Documents (embeddings_clinical_medium) author: John Snow Labs name: ner_vop_demographic_emb_clinical_medium date: 2023-06-06 tags: [licensed, clinical, ner, en, vop, demographic] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model extracts demographic terms from the documents transferred from the patient’s own sentences. ## Predicted Entities `Gender`, `Employment`, `RaceEthnicity`, `Age`, `Substance`, `RelationshipStatus`, `SubstanceQuantity` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/VOICE_OF_THE_PATIENTS/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_vop_demographic_emb_clinical_medium_en_4.4.3_3.0_1686075342272.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_vop_demographic_emb_clinical_medium_en_4.4.3_3.0_1686075342272.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | |:--------|:--------------| | grandma | Gender | | 85 | Age | | Black | RaceEthnicity | | doctors | Employment | | her | Gender | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_vop_demographic_emb_clinical_medium| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|3.8 MB| |Dependencies:|embeddings_clinical_medium| ## References In-house annotated health-related text in colloquial language. ## Sample text from the training dataset Hello,I'm 20 year old girl. I'm diagnosed with hyperthyroid 1 month ago. I was feeling weak, light headed,poor digestion, panic attacks, depression, left chest pain, increased heart rate, rapidly weight loss, from 4 months. Because of this, I stayed in the hospital and just discharged from hospital. I had many other blood tests, brain mri, ultrasound scan, endoscopy because of some dumb doctors bcs they were not able to diagnose actual problem. Finally I got an appointment with a homeopathy doctor finally he find that i was suffering from hyperthyroid and my TSH was 0.15 T3 and T4 is normal . Also i have b12 deficiency and vitamin D deficiency so I'm taking weekly supplement of vitamin D and 1000 mcg b12 daily. I'm taking homeopathy medicine for 40 days and took 2nd test after 30 days. My TSH is 0.5 now. I feel a little bit relief from weakness and depression but I'm facing with 2 new problem from last week that is breathtaking problem and very rapid heartrate. I just want to know if i should start allopathy medicine or homeopathy is okay? Bcs i heard that thyroid take time to start recover. So please let me know if both of medicines take same time. Because some of my friends advising me to start allopathy and never take a chance as i can develop some serious problems.Sorry for my poor english😐Thank you. ## Benchmarking ```bash label tp fp fn total precision recall f1 Gender 1299 23 18 1317 0.98 0.99 0.98 Employment 1175 45 68 1243 0.96 0.95 0.95 RaceEthnicity 31 2 2 33 0.94 0.94 0.94 Age 546 54 36 582 0.91 0.94 0.92 Substance 389 48 32 421 0.89 0.92 0.91 RelationshipStatus 21 5 3 24 0.81 0.88 0.84 SubstanceQuantity 65 18 20 85 0.78 0.76 0.77 macro_avg 3526 195 179 3705 0.90 0.91 0.90 micro_avg 3526 195 179 3705 0.95 0.95 0.95 ``` --- layout: model title: Detect Social Determinants of Health Mentions author: John Snow Labs name: ner_sdoh_mentions date: 2022-12-18 tags: [en, licensed, ner, sdoh, mentions, clinical] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 4.2.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Named Entity Recognition model is intended for detecting Social Determinants of Health mentions in clinical notes and trained by using MedicalNerApproach annotator that allows to train generic NER models based on Neural Networks. | Entitiy Name | Descriptions | Sample Texts | chunks+labels | |------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| | sdoh_community | The patient's social
and community networks, including family members,
friends, and other social connections. | - He has a 27 yo son.
- The patient lives with mother.
- She is a widow.
- Married and has two children.
| - (son),(sdoh_community)
- (mother),(sdoh_community)
- (widow),(sdoh_community)
- (Married, children),(sdoh_community,sdoh_community)
| | sdoh_economics | The patient's economic
status and financial resources, including their
occupation, income, and employment status. | - The patient worked as an accountant.
- He is a retired history professor.
- She is a lawyer.
- Worked in insurance, currently unemployed.
| - (worked),(sdoh_economics)
- (retired),(sdoh_economics)
- (lawyer),(sdoh_economics)
- (worked, unemployed),(sdoh_economics, sdoh_economics) | | sdoh_education | The patient's education-related
passages such as schooling, college, or degrees attained. | - She graduated from high school.
- He is a fourth grade teacher in inner city.
- He completed some college.
| - (graduated from high school),(sdoh_education)
- (teacher),(sdoh_education)
- (college),(sdoh_education)
| | sdoh_environment | The patient's living
environment and access to housing. | - He lives at home.
- Her other daughter lives in the apartment below.
- The patient lives with her husband in a retirement community.
| - (home),(sdoh_environment)
- (apartment),(sdoh_environment)
- (retirement community),(sdoh_environment)
| | behavior_tobacco | This entity is labeled based on any indication of
the patient's current or past tobacco
use and smoking history | - She smoked one pack a day for forty years.
- The patient denies tobacco use.
- The patient smokes an occasional cigar.
| - (smoked one pack),(behavior_tobacco)
- (tobacco),(behavior_tobacco)
- (smokes an occasional cigar),(behavior_tobacco)
| | behavior_alcohol | This entity is used to label indications of
the patient's alcohol consumption. | - She drinks alcohol.
- The patient denies ethanol.
- He denies ETOH.
| - (alcohol),(behavior_alcohol)
- (ethanol),(behavior_alcohol)
- (ETOH),(behavior_alcohol)
| | behavior_drug | This entity is used to label any indications of
the patient's current or past drug use. | - She denies any intravenous drug abuse.
- No illicit drug use including IV per family.
- The patient any using recreational drugs.
| - (intravenous drug),(behavior_drug)
- (illicit drug, IV),(behavior_drug, behavior_drug)
- (recreational drugs),(behavior_drug)
| ## Predicted Entities `sdoh_community`, `sdoh_economics`, `sdoh_education`, `sdoh_environment`, `behavior_tobacco`, `behavior_alcohol`, `behavior_drug` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_mentions_en_4.2.2_3.0_1671369830893.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_sdoh_mentions_en_4.2.2_3.0_1671369830893.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------+----------------+ |chunk |ner_label | +----------------+----------------+ |married |sdoh_community | |children |sdoh_community | |works |sdoh_economics | |alcohol |behavior_alcohol| |intravenous drug|behavior_drug | |smoking |behavior_tobacco| +----------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_sdoh_mentions| |Compatibility:|Healthcare NLP 4.2.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|15.1 MB| ## Benchmarking ```bash label precision recall f1-score support behavior_alcohol 0.95 0.94 0.94 798 behavior_drug 0.93 0.92 0.92 366 behavior_tobacco 0.95 0.95 0.95 936 sdoh_community 0.97 0.97 0.97 969 sdoh_economics 0.95 0.91 0.93 363 sdoh_education 0.69 0.65 0.67 34 sdoh_environment 0.93 0.90 0.92 651 micro-avg 0.95 0.94 0.94 4117 macro-avg 0.91 0.89 0.90 4117 weighted-avg 0.95 0.94 0.94 4117 ``` --- layout: model title: Castilian, Spanish BertForQuestionAnswering model (from CenIA) author: John Snow Labs name: bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: es edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-spanish-wwm-uncased-finetuned-qa-tar` is a Castilian, Spanish model orginally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar_es_4.0.0_3.0_1654180605963.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar_es_4.0.0_3.0_1654180605963.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar","es") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("es.answer_question.bert.base_uncased.by_CenIA").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_spanish_wwm_uncased_finetuned_qa_tar| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|410.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/bert-base-spanish-wwm-uncased-finetuned-qa-tar --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Spanish (WikiNER 840B 300) author: John Snow Labs name: wikiner_840B_300 date: 2020-02-03 task: Named Entity Recognition language: es edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [ner, es, open_source] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 840B 300 is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_ES){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_ES.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_es_2.4.0_2.4_1581971942091.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_es_2.4.0_2.4_1581971942091.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-------------------------------------------------------------------+---------+ |chunk |ner_label| +-------------------------------------------------------------------+---------+ |William Henry Gates III |PER | |Microsoft Corporation |ORG | |Durante su carrera |MISC | |Microsoft |ORG | |Gates |PER | |Nacido |MISC | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |Nuevo México |LOC | |Gates |PER | |CEO |ORG | |A fines de la década de 1990 |MISC | |Gates |PER | |Esta opinión ha sido confirmada por numerosas sentencias judiciales|MISC | |Gates |PER | |Microsoft |ORG | +-------------------------------------------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|es| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://es.wikipedia.org](https://es.wikipedia.org) --- layout: model title: GPT2 text-to-text model (Large) author: John Snow Labs name: gpt_large date: 2021-12-03 tags: [en, open_source] task: Text Generation language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: GPT2Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description GPT-2 displays a broad set of capabilities, including the ability to generate conditional synthetic text samples of unprecedented quality, where the model is primed with an input and it generates a lengthy continuation. This is the large model (bigger than Base). Other models (medium, base) are also available in [Models Hub](https://nlp.johnsnowlabs.com/models?task=Text+Generation) ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/gpt_large_en_3.4.0_3.0_1638547401185.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/gpt_large_en_3.4.0_3.0_1638547401185.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \\ .setInputCol("text") \\ .setOutputCol("documents") gpt2 = GPT2Transformer.pretrained("gpt2_large") \\ .setInputCols(["documents"]) \\ .setMaxOutputLength(50) \\ .setOutputCol("generation") pipeline = Pipeline().setStages([documentAssembler, gpt2]) data = spark.createDataFrame([["My name is Leonardo."]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("summaries.generation").show(truncate=False) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val gpt2 = GPT2Transformer.pretrained("gpt2_large") .setInputCols(Array("documents")) .setMinOutputLength(10) .setMaxOutputLength(50) .setDoSample(false) .setTopK(50) .setNoRepeatNgramSize(3) .setOutputCol("generation") val pipeline = new Pipeline().setStages(Array(documentAssembler, gpt2)) val data = Seq("My name is Leonardo.").toDF("text") val result = pipeline.fit(data).transform(data) results.select("generation.result").show(truncate = false) ``` {:.nlu-block} ```python import nlu nlu.load("en.gpt2.large").predict("""My name is Leonardo.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|gpt_large| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[generation]| |Language:|en| ## Data Source OpenAI WebText - a corpus created by scraping web pages with emphasis on document quality. It consists of over 8 million documents for a total of 40 GB of text. All Wikipedia documents were removed from WebText. --- layout: model title: English AlbertForQuestionAnswering model (from SalmanMo) author: John Snow Labs name: albert_qa_QA_1e date: 2022-06-24 tags: [en, open_source, albert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: AlBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ALBERT_QA_1e` is a English model originally trained by `SalmanMo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_qa_QA_1e_en_4.0.0_3.0_1656062921751.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_qa_QA_1e_en_4.0.0_3.0_1656062921751.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = AlbertForQuestionAnswering.pretrained("albert_qa_QA_1e","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = AlbertForQuestionAnswering.pretrained("albert_qa_QA_1e","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.albert.by_SalmanMo").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_qa_QA_1e| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|42.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SalmanMo/ALBERT_QA_1e --- layout: model title: English image_classifier_vit_pond_image_classification_4 ViTForImageClassification from SummerChiam author: John Snow Labs name: image_classifier_vit_pond_image_classification_4 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_pond_image_classification_4` is a English model originally trained by SummerChiam. ## Predicted Entities `Normal`, `Boiling`, `Algae`, `NormalCement`, `NormalRain`, `BoilingNight`, `NormalNight` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_4_en_4.1.0_3.0_1660166981854.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_pond_image_classification_4_en_4.1.0_3.0_1660166981854.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_pond_image_classification_4", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_pond_image_classification_4", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_pond_image_classification_4| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English Bert Embeddings (Large, Cased) author: John Snow Labs name: bert_embeddings_bert_large_cased_whole_word_masking date: 2022-04-11 tags: [bert, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-large-cased-whole-word-masking` is a English model orginally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_cased_whole_word_masking_en_3.4.2_3.0_1649671766081.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_large_cased_whole_word_masking_en_3.4.2_3.0_1649671766081.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_large_cased_whole_word_masking| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/bert-large-cased-whole-word-masking - https://arxiv.org/abs/1810.04805 - https://github.com/google-research/bert - https://yknzhu.wixsite.com/mbweb - https://en.wikipedia.org/wiki/English_Wikipedia --- layout: model title: English T5ForConditionalGeneration Cased model (from KES) author: John Snow Labs name: t5_ttparser date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `T5-TTParser` is a English model originally trained by `KES`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_ttparser_en_4.3.0_3.0_1675099426092.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_ttparser_en_4.3.0_3.0_1675099426092.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_ttparser","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_ttparser","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_ttparser| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|916.4 MB| ## References - https://huggingface.co/KES/T5-TTParser - https://arxiv.org/abs/1702.04066 - https://pypi.org/project/Caribe/ --- layout: model title: English DistilBertForQuestionAnswering Base Cased model (from Moussab) author: John Snow Labs name: distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-cased-distilled-squad-orkg-no-label-1e-4` is a English model originally trained by `Moussab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4_en_4.3.0_3.0_1672766759116.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4_en_4.3.0_3.0_1672766759116.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_cased_led_squad_orkg_no_label_1e_4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|244.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Moussab/distilbert-base-cased-distilled-squad-orkg-no-label-1e-4 --- layout: model title: Detect Assertion Status from Oncology Tests author: John Snow Labs name: assertion_oncology_test_binary_wip date: 2022-10-01 tags: [licensed, clinical, oncology, en, assertion, test, biomarker] task: Assertion Status language: en nav_key: models edition: Healthcare NLP 4.1.0 spark_version: 3.0 supported: true annotator: AssertionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model detects the assertion status of oncology tests, such as Pathology_Test or Imaging_Test. The model identifies if the test has been performed (Present_Or_Past status), or if it is mentioned but in hypothetical or absent sentences (Hypothetical_Or_Absent status). ## Predicted Entities `Hypothetical_Or_Absent`, `Medical_History` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ASSERTION_ONCOLOGY/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/27.Oncology_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_test_binary_wip_en_4.1.0_3.0_1664641843088.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/assertion_oncology_test_binary_wip_en_4.1.0_3.0_1664641843088.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | chunk | ner_label | assertion | |:--------|:---------------|:-----------------------| | biopsy | Pathology_Test | Medical_History | | CT scan | Imaging_Test | Hypothetical_Or_Absent | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|assertion_oncology_test_binary_wip| |Compatibility:|Healthcare NLP 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion_pred]| |Language:|en| |Size:|1.4 MB| ## References In-house annotated oncology case reports. ## Benchmarking ```bash label precision recall f1-score support Hypothetical_Or_Absent 0.79 0.81 0.80 37.0 Medical_History 0.80 0.78 0.79 36.0 macro-avg 0.79 0.79 0.79 73.0 weighted-avg 0.79 0.79 0.79 73.0 ``` --- layout: model title: Spanish DistilBertForQuestionAnswering Base Uncased model (from CenIA) author: John Snow Labs name: distilbert_qa_distillbert_base_uncased_finetuned_tar date: 2023-01-03 tags: [es, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distillbert-base-spanish-uncased-finetuned-qa-tar` is a Spanish model originally trained by `CenIA`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_uncased_finetuned_tar_es_4.3.0_3.0_1672774812379.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_distillbert_base_uncased_finetuned_tar_es_4.3.0_3.0_1672774812379.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_uncased_finetuned_tar","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distillbert_base_uncased_finetuned_tar","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distillbert_base_uncased_finetuned_tar| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|es| |Size:|250.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/CenIA/distillbert-base-spanish-uncased-finetuned-qa-tar --- layout: model title: English BertForQuestionAnswering model (from zhufy) author: John Snow Labs name: bert_qa_squad_en_bert_base date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squad-en-bert-base` is a English model orginally trained by `zhufy`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_squad_en_bert_base_en_4.0.0_3.0_1654191955457.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_squad_en_bert_base_en_4.0.0_3.0_1654191955457.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_squad_en_bert_base","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_squad_en_bert_base","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base.by_zhufy").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_squad_en_bert_base| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/zhufy/squad-en-bert-base - https://rajpurkar.github.io/SQuAD-explorer/ --- layout: model title: Chinese Bert Embeddings (from amine) author: John Snow Labs name: bert_embeddings_bert_base_5lang_cased date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-5lang-cased` is a Chinese model orginally trained by `amine`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_5lang_cased_zh_3.4.2_3.0_1649669822302.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_5lang_cased_zh_3.4.2_3.0_1649669822302.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_5lang_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|464.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/amine/bert-base-5lang-cased - https://cloud.google.com/compute/docs/machine-types#n1_machine_type --- layout: model title: News Classifier for Urdu texts author: John Snow Labs name: classifierdl_bert_news date: 2021-12-10 tags: [urdu, news, classifier, ur, open_source] task: Text Classification language: ur edition: Spark NLP 3.3.0 spark_version: 2.4 supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify Urdu news into 7 categories. ## Predicted Entities `business`, `entertainment`, `health`, `inland`, `science`, `sports`, `weird_news` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/CLASSIFICATION_UR_NEWS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/CLASSIFICATION_UR_NEWS.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_bert_news_ur_3.3.0_2.4_1639125233132.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_bert_news_ur_3.3.0_2.4_1639125233132.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("news") \ .setOutputCol("document") embeddings = BertSentenceEmbeddings.pretrained("labse", "xx") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") classifierdl = ClassifierDLModel.pretrained("classifierdl_bert_news", "ur") \ .setInputCols(["document", "sentence_embeddings"]) \ .setOutputCol("class") urdu_news_pipeline = Pipeline(stages=[document_assembler, embeddings, classifierdl]) light_pipeline = LightPipeline(urdu_news_pipeline.fit(spark.createDataFrame([['']]).toDF("news"))) result = light_pipeline.annotate("گزشتہ ہفتے ایپل کے حصص میں 11 فیصد اضافہ ہوا ہے۔") result["class"] ``` ```scala val document = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val embeddings = BertSentenceEmbeddings .pretrained("lanse", "xx") .setInputCols("document") .setOutputCol("sentence_embeddings") val document_classifier = ClassifierDLModel.pretrained("classifierdl_bert_news", "ur") .setInputCols(Array("document", "sentence_embeddings")) .setOutputCol("class") val nlpPipeline = new Pipeline().setStages(Array(document, embeddings, document_classifier)) val light_pipeline = LightPipeline(nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) val result = light_pipeline.annotate("گزشتہ ہفتے ایپل کے حصص میں 11 فیصد اضافہ ہوا ہے۔") ``` {:.nlu-block} ```python import nlu nlu.load("ur.classify.news").predict("""گزشتہ ہفتے ایپل کے حصص میں 11 فیصد اضافہ ہوا ہے۔""") ```

## Results ```bash ['business'] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_bert_news| |Compatibility:|Spark NLP 3.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|ur| |Size:|23.6 MB| ## Data Source Combination of multiple open source data sets. ## Benchmarking ```bash label precision recall f1-score support business 0.83 0.86 0.85 2365 entertainment 0.87 0.85 0.86 3081 health 0.68 0.67 0.68 430 inland 0.80 0.82 0.81 3964 science 0.62 0.60 0.61 558 sports 0.88 0.89 0.89 4022 weird_news 0.60 0.54 0.57 826 accuracy - - 0.82 15246 macro-avg 0.76 0.75 0.75 15246 weighted-avg 0.82 0.82 0.82 15246 ``` --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from robkayinto) author: John Snow Labs name: xlmroberta_ner_robkayinto_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `robkayinto`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_robkayinto_base_finetuned_panx_all_xx_4.1.0_3.0_1660428813467.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_robkayinto_base_finetuned_panx_all_xx_4.1.0_3.0_1660428813467.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_robkayinto_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_robkayinto_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_robkayinto_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/robkayinto/xlm-roberta-base-finetuned-panx-all --- layout: model title: English RobertaForQuestionAnswering (from mrm8488) author: John Snow Labs name: roberta_qa_distilroberta_finetuned_squadv1 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilroberta-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_finetuned_squadv1_en_4.0.0_3.0_1655728443758.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_distilroberta_finetuned_squadv1_en_4.0.0_3.0_1655728443758.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_distilroberta_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_distilroberta_finetuned_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.distilled").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_distilroberta_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|307.0 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/distilroberta-finetuned-squadv1 --- layout: model title: English asr_wav2vec2_base_timit_demo_colab92 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab92 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab92` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab92_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab92_en_4.2.0_3.0_1664022445550.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab92_en_4.2.0_3.0_1664022445550.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab92', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab92", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab92| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Indonesian T5ForConditionalGeneration Base Cased model (from panggi) author: John Snow Labs name: t5_panggi_base_indonesian_summarization_cased date: 2023-01-30 tags: [id, open_source, t5, tensorflow] task: Text Generation language: id edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-base-indonesian-summarization-cased` is a Indonesian model originally trained by `panggi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_panggi_base_indonesian_summarization_cased_id_4.3.0_3.0_1675109758797.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_panggi_base_indonesian_summarization_cased_id_4.3.0_3.0_1675109758797.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_panggi_base_indonesian_summarization_cased","id") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_panggi_base_indonesian_summarization_cased","id") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_panggi_base_indonesian_summarization_cased| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|id| |Size:|925.6 MB| ## References - https://huggingface.co/panggi/t5-base-indonesian-summarization-cased - https://github.com/kata-ai/indosum --- layout: model title: Chinese BertForQuestionAnswering model (from hfl) author: John Snow Labs name: bert_qa_chinese_pert_large_mrc date: 2022-06-06 tags: [zh, open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese-pert-large-mrc` is a Chinese model orginally trained by `hfl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_chinese_pert_large_mrc_zh_4.0.0_3.0_1654537830236.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_chinese_pert_large_mrc_zh_4.0.0_3.0_1654537830236.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_chinese_pert_large_mrc","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_chinese_pert_large_mrc","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert.large.by_hfl").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_chinese_pert_large_mrc| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/hfl/chinese-pert-large-mrc - https://github.com/ymcui/PERT - https://github.com/ymcui/Chinese-ELECTRA - https://github.com/ymcui/Chinese-Minority-PLM - https://github.com/ymcui/HFL-Anthology - https://github.com/ymcui/Chinese-BERT-wwm - https://github.com/ymcui/Chinese-XLNet - https://github.com/airaria/TextBrewer - https://github.com/ymcui/MacBERT --- layout: model title: Pipeline to Detect Chemicals and Proteins in text author: John Snow Labs name: ner_chemprot_biobert_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chemprot_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_chemprot_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_biobert_pipeline_en_3.4.1_3.0_1647868851526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemprot_biobert_pipeline_en_3.4.1_3.0_1647868851526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_chemprot_biobert_pipeline", "en", "clinical/models") pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` ```scala val pipeline = new PretrainedPipeline("ner_chemprot_biobert_pipeline", "en", "clinical/models") pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemprot_biobert.pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

## Results ```bash +-------------------------------+--------+ |chunks |entities| +-------------------------------+--------+ |Keratinocyte growth factor |GENE-Y | |acidic fibroblast growth factor|GENE-Y | +-------------------------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemprot_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.0 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter --- layout: model title: Legal Optional renewal Clause Binary Classifier author: John Snow Labs name: legclf_optional_renewal_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `optional_renewal` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `optional_renewal` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_optional_renewal_clause_en_1.0.0_3.2_1660122788779.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_optional_renewal_clause_en_1.0.0_3.2_1660122788779.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_optional_renewal_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[optional_renewal]| |[other]| |[other]| |[optional_renewal]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_optional_renewal_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support optional_renewal 1.00 0.91 0.95 32 other 0.97 1.00 0.99 101 accuracy - - 0.98 133 macro-avg 0.99 0.95 0.97 133 weighted-avg 0.98 0.98 0.98 133 ``` --- layout: model title: English asr_wav2vec2_large_xls_r_300m_urdu_proj TFWav2Vec2ForCTC from MSaudTahir author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_urdu_proj date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_urdu_proj` is a English model originally trained by MSaudTahir. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_urdu_proj_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_urdu_proj_en_4.2.0_3.0_1664101958745.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_urdu_proj_en_4.2.0_3.0_1664101958745.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_urdu_proj", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_urdu_proj", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_urdu_proj| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Legal Acceleration Clause Binary Classifier author: John Snow Labs name: legclf_acceleration_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `acceleration` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `acceleration` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_acceleration_clause_en_1.0.0_3.2_1660123216364.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_acceleration_clause_en_1.0.0_3.2_1660123216364.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_acceleration_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[acceleration]| |[other]| |[other]| |[acceleration]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_acceleration_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support acceleration 1.00 0.78 0.88 27 other 0.92 1.00 0.96 65 accuracy - - 0.93 92 macro-avg 0.96 0.89 0.92 92 weighted-avg 0.94 0.93 0.93 92 ``` --- layout: model title: Financial Relation Extraction (Tickers) author: John Snow Labs name: finre_has_ticker date: 2022-10-15 tags: [en, finance, re, has_ticker, licensed] task: Relation Extraction language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to extract the Ticker of Companies or Product names. A Ticker (stock symbol) is a unique series of letters assigned to a security for trading purposes. For example: Company: Apple Inc. Ticker: `AAPL` ## Predicted Entities `has_ticker` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finre_has_ticker_en_1.0.0_3.0_1665842119957.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finre_has_ticker_en_1.0.0_3.0_1665842119957.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl","xx")\ .setInputCols(["document"])\ .setOutputCol("sentence")\ tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model_org = finance.NerModel.pretrained("finner_orgs_prods_alias","en","finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner_org") ner_converter_org = nlp.NerConverter()\ .setInputCols(["sentence","token","ner_org"])\ .setOutputCol("ner_chunk_org")\ .setWhiteList(['ORG']) ner_model_ticker = finance.NerModel.pretrained("finner_ticker", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner_ticker") ner_converter_ticker = nlp.NerConverter() \ .setInputCols(["sentence", "token", "ner_ticker"]) \ .setOutputCol("ner_chunk_ticker") chunk_merger = finance.ChunkMergeApproach()\ .setInputCols("ner_chunk_ticker", "ner_chunk_org")\ .setOutputCol('ner_chunk')\ .setMergeOverlapping(True) pos = nlp.PerceptronModel.pretrained("pos_anc", 'en')\ .setInputCols("sentence", "token")\ .setOutputCol("pos") dependency_parser = nlp.DependencyParserModel().pretrained("dependency_conllu", "en")\ .setInputCols(["sentence", "pos", "token"])\ .setOutputCol("dependencies") re_ner_chunk_filter = finance.RENerChunksFilter()\ .setInputCols(["ner_chunk", "dependencies"])\ .setOutputCol("re_ner_chunk")\ .setRelationPairs(["ORG-TICKER"])\ .setMaxSyntacticDistance(4) re_Model = finance.RelationExtractionDLModel.pretrained("finre_has_ticker", "en", "finance/models")\ .setInputCols(["ner_chunk", "sentence"])\ .setOutputCol("relations")\ .setPredictionThreshold(0.2) pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model_org, ner_converter_org, ner_model_ticker, ner_converter_ticker, chunk_merger, pos, dependency_parser, re_ner_chunk_filter, re_Model]) empty_df = spark.createDataFrame([['']]).toDF("text") re_model = pipeline.fit(empty_df) text="""'MTH - Meritage Homes Corporation Reports Disappointing Revenue. RECN, Resources Connection Inc. Shareholder Raymond James Trust Has Decreased Holding'""" light_model = nlp.LightPipeline(re_model) light_model.fullAnnotate(text) ```

## Results ```bash | relation | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_begin | entity2_end | chunk2 | confidence | |-----------:|--------:|--------------:|------------:|-------:|--------:|--------------:|------------:|---------------------------:|-----------:| | has_ticker | TICKER | 0 | 2 | MTH | ORG | 6 | 31 | Meritage Homes Corporation | 0.99532026 | | has_ticker | TICKER | 64 | 67 | RECN | ORG | 70 | 93 | Resources Connection Inc | 0.97409964 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finre_has_ticker| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|409.9 MB| ## References Manual annotations on tweets ## Benchmarking ```bash label Recall Precision F1 Support has_ticker 0.717 0.827 0.768 60 Avg. 0.717 0.827 0.768 - Weighted-Avg. 0.717 0.827 0.768 - ``` --- layout: model title: Translate English to Albanian Pipeline author: John Snow Labs name: translate_en_sq date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, sq, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `sq` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_sq_xx_2.7.0_2.4_1609690542107.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_sq_xx_2.7.0_2.4_1609690542107.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_sq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_sq", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.sq').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_sq| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Kinyarwanda XLMRobertaForTokenClassification Base Cased model (from mbeukman) author: John Snow Labs name: xlmroberta_ner_base_finetuned_ner_kinyarwand date: 2022-08-01 tags: [rw, open_source, xlm_roberta, ner] task: Named Entity Recognition language: rw edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-ner-kinyarwanda` is a Kinyarwanda model originally trained by `mbeukman`. ## Predicted Entities `DATE`, `PER`, `ORG`, `LOC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659354904793.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_base_finetuned_ner_kinyarwand_rw_4.1.0_3.0_1659354904793.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_ner_kinyarwand","rw") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_base_finetuned_ner_kinyarwand","rw") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_base_finetuned_ner_kinyarwand| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|rw| |Size:|775.8 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/mbeukman/xlm-roberta-base-finetuned-ner-kinyarwanda - https://arxiv.org/abs/2103.11811 - https://github.com/Michael-Beukman/NERTransfer - https://github.com/masakhane-io/masakhane-ner --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-128-finetuned-squad-seed-10` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10_en_4.0.0_3.0_1654191366585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10_en_4.0.0_3.0_1654191366585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_128d_seed_10").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_128_finetuned_squad_seed_10| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|381.1 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-128-finetuned-squad-seed-10 --- layout: model title: English BertForQuestionAnswering Cased model (from LeoFelix) author: John Snow Labs name: bert_qa_leofelix_finetuned_squad date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model originally trained by `LeoFelix`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_leofelix_finetuned_squad_en_4.0.0_3.0_1657186109766.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_leofelix_finetuned_squad_en_4.0.0_3.0_1657186109766.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_leofelix_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_leofelix_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_leofelix_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|406.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LeoFelix/bert-finetuned-squad --- layout: model title: Smaller BERT Embeddings (L-8_H-512_A-8) author: John Snow Labs name: small_bert_L8_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L8_512_en_2.6.0_2.4_1598344705269.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L8_512_en_2.6.0_2.4_1598344705269.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L8_512", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L8_512", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L8_512').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L8_512_embeddings I [-0.0785166472196579, -0.22764667868614197, 0.... love [0.4189109206199646, 0.17259590327739716, 0.87... NLP [0.583548367023468, -0.21391963958740234, -0.2... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L8_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1 --- layout: model title: Yue Chinese asr_wav2vec2_large_xlsr_cantonese_by_ctl TFWav2Vec2ForCTC from ctl author: John Snow Labs name: pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl date: 2022-09-24 tags: [wav2vec2, yue, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: yue edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_cantonese_by_ctl` is a Yue Chinese model originally trained by ctl. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl_yue_4.2.0_3.0_1664039821455.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl_yue_4.2.0_3.0_1664039821455.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl', lang = 'yue') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl", lang = "yue") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_large_xlsr_cantonese_by_ctl| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|yue| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: German BertForQuestionAnswering model (from Sahajtomar) author: John Snow Labs name: bert_qa_GBERTQnA date: 2022-06-02 tags: [de, open_source, question_answering, bert] task: Question Answering language: de edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `GBERTQnA` is a German model orginally trained by `Sahajtomar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_GBERTQnA_de_4.0.0_3.0_1654176679493.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_GBERTQnA_de_4.0.0_3.0_1654176679493.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_GBERTQnA","de") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_GBERTQnA","de") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("de.answer_question.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_GBERTQnA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Sahajtomar/GBERTQnA - https://github.com/Sahajtomar/Question-Answering/blob/main/Sahajtomar_GBERTQnA.ipynb --- layout: model title: Translate Afrikaans to English Pipeline author: John Snow Labs name: translate_af_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, af, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `af` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_af_en_xx_2.7.0_2.4_1609687489458.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_af_en_xx_2.7.0_2.4_1609687489458.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_af_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_af_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.af.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_af_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Fast Neural Machine Translation Model from Pangasinan to English author: John Snow Labs name: opus_mt_pag_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pag, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `pag` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_pag_en_xx_2.7.0_2.4_1609167115456.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_pag_en_xx_2.7.0_2.4_1609167115456.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_pag_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English DistilBertForQuestionAnswering model (from LucasS) author: John Snow Labs name: distilbert_qa_distilBertABSA date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilBertABSA` is a English model originally trained by `LucasS`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_distilBertABSA_en_4.0.0_3.0_1654723518164.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_distilBertABSA_en_4.0.0_3.0_1654723518164.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distilBertABSA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distilBertABSA","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.by_LucasS").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distilBertABSA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/LucasS/distilBertABSA --- layout: model title: Marathi DistilBERT Embeddings (from DarshanDeshpande) author: John Snow Labs name: distilbert_embeddings_marathi_distilbert date: 2022-04-12 tags: [distilbert, embeddings, mr, open_source] task: Embeddings language: mr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `marathi-distilbert` is a Marathi model orginally trained by `DarshanDeshpande`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_marathi_distilbert_mr_3.4.2_3.0_1649783605801.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_marathi_distilbert_mr_3.4.2_3.0_1649783605801.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_marathi_distilbert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|mr| |Size:|247.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/DarshanDeshpande/marathi-distilbert - https://github.com/DarshanDeshpande - https://www.linkedin.com/in/darshan-deshpande/ - https://github.com/Baras64 - http://www.linkedin.com/in/harsh-abhi --- layout: model title: English Named Entity Recognition (from sven-nm) author: John Snow Labs name: roberta_ner_roberta_classics_ner date: 2022-05-03 tags: [roberta, ner, open_source, en] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta_classics_ner` is a English model orginally trained by `sven-nm`. ## Predicted Entities `REFSCOPE`, `AWORK`, `FRAGREF`, `AAUTHOR`, `REFAUWORK` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_classics_ner_en_3.4.2_3.0_1651594279156.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_ner_roberta_classics_ner_en_3.4.2_3.0_1651594279156.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_ner_roberta_classics_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|444.3 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/sven-nm/roberta_classics_ner - https://www.epische-bauformen.uni-rostock.de/ - http://infoscience.epfl.ch/record/291236?&ln=en - https://github.com/impresso/CLEF-HIPE-2020-scorer - https://github.com/AjaxMultiCommentary --- layout: model title: Detect Cancer Genetics author: John Snow Labs name: ner_bionlp date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for biology and genetics terms. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities ``Amino_acid``, ``Anatomical_system``, ``Cancer``, ``Cell``, ``Cellular_component``, ``Developing_anatomical_Structure``, ``Gene_or_gene_product``, ``Immaterial_anatomical_entity``, ``Multi-tissue_structure``, ``Organ``, ``Organism``, ``Organism_subdivision``, ``Simple_chemical``, ``Tissue`` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_TUMOR){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_en_3.0.0_3.0_1617208458431.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bionlp_en_3.0.0_3.0_1617208458431.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_bionlp", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes."""]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_bionlp", "en", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.bionlp").predict("""The human KCNJ9 (Kir 3.3, GIRK3) is a member of the G-protein-activated inwardly rectifying potassium (GIRK) channel family. Here we describe the genomicorganization of the KCNJ9 locus on chromosome 1q21-23 as a candidate gene forType II diabetes mellitus in the Pima Indian population. The gene spansapproximately 7.6 kb and contains one noncoding and two coding exons separated byapproximately 2.2 and approximately 2.6 kb introns, respectively. We identified14 single nucleotide polymorphisms (SNPs), including one that predicts aVal366Ala substitution, and an 8 base-pair (bp) insertion/deletion. Ourexpression studies revealed the presence of the transcript in various humantissues including pancreas, and two major insulin-responsive tissues: fat andskeletal muscle. The characterization of the KCNJ9 gene should facilitate furtherstudies on the function of the KCNJ9 protein and allow evaluation of thepotential role of the locus in Type II diabetes.""") ```

## Results ```bash |id |sentence_id|chunk |begin|end|ner_label | +---+-----------+----------------------+-----+---+--------------------+ |0 |0 |human |4 |8 |Organism | |0 |0 |Kir 3.3 |17 |23 |Gene_or_gene_product| |0 |0 |GIRK3 |26 |30 |Gene_or_gene_product| |0 |0 |potassium |92 |100|Simple_chemical | |0 |0 |GIRK |103 |106|Gene_or_gene_product| |0 |1 |chromosome 1q21-23 |188 |205|Cellular_component | |0 |5 |pancreas |697 |704|Organ | |0 |5 |tissues |740 |746|Tissue | |0 |5 |fat andskeletal muscle|749 |770|Tissue | |0 |6 |KCNJ9 |801 |805|Gene_or_gene_product| |0 |6 |Type II |940 |946|Gene_or_gene_product| |1 |0 |breast cancer |84 |96 |Cancer | |1 |0 |patients |134 |141|Organism | |1 |0 |anthracyclines |167 |180|Simple_chemical | |1 |0 |taxanes |186 |192|Simple_chemical | |1 |1 |vinorelbine |246 |256|Simple_chemical | |1 |1 |patients |273 |280|Organism | |1 |1 |breast |309 |314|Cancer | |1 |1 |vinorelbine inpatients|386 |407|Simple_chemical | |1 |1 |anthracyclines |433 |446|Simple_chemical | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bionlp| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on Cancer Genetics (CG) task of the BioNLP Shared Task 2013 with ``embeddings_clinical``. https://aclanthology.org/W13-2008/ ## Benchmarking ```bash | | label | tp | fp | fn | prec | rec | f1 | |---:|:----------------------------------|------:|------:|-----:|---------:|---------:|---------:| | 1 | I-Amino_acid | 1 | 0 | 2 | 1 | 0.333333 | 0.5 | | 2 | I-Simple_chemical | 264 | 39 | 358 | 0.871287 | 0.424437 | 0.570811 | | 3 | B-Immaterial_anatomical_entity | 19 | 12 | 12 | 0.612903 | 0.612903 | 0.612903 | | 4 | B-Cellular_component | 144 | 24 | 36 | 0.857143 | 0.8 | 0.827586 | | 5 | B-Cancer | 808 | 103 | 115 | 0.886937 | 0.875406 | 0.881134 | | 6 | I-Cell | 888 | 91 | 198 | 0.907048 | 0.81768 | 0.860048 | | 7 | B-Tissue | 137 | 44 | 47 | 0.756906 | 0.744565 | 0.750685 | | 8 | B-Organism_substance | 67 | 4 | 34 | 0.943662 | 0.663366 | 0.77907 | | 9 | B-Simple_chemical | 598 | 165 | 128 | 0.783748 | 0.823692 | 0.803224 | | 10 | B-Cell | 910 | 125 | 98 | 0.879227 | 0.902778 | 0.890847 | | 11 | I-Organ | 7 | 2 | 10 | 0.777778 | 0.411765 | 0.538462 | | 12 | I-Tissue | 86 | 21 | 25 | 0.803738 | 0.774775 | 0.788991 | | 13 | I-Pathological_formation | 20 | 5 | 19 | 0.8 | 0.512821 | 0.625 | | 14 | I-Organism | 58 | 13 | 62 | 0.816901 | 0.483333 | 0.60733 | | 15 | B-Gene_or_gene_product | 2354 | 282 | 165 | 0.89302 | 0.934498 | 0.913288 | | 16 | I-Cancer | 488 | 73 | 116 | 0.869875 | 0.807947 | 0.837768 | | 17 | B-Organ | 109 | 36 | 47 | 0.751724 | 0.698718 | 0.724252 | | 18 | B-Pathological_formation | 58 | 20 | 30 | 0.74359 | 0.659091 | 0.698795 | | 19 | I-Cellular_component | 33 | 5 | 36 | 0.868421 | 0.478261 | 0.616822 | | 20 | I-Multi-tissue_structure | 132 | 34 | 29 | 0.795181 | 0.819876 | 0.807339 | | 21 | B-Organism | 437 | 53 | 77 | 0.891837 | 0.850195 | 0.870518 | | 22 | I-Gene_or_gene_product | 1268 | 161 | 1086 | 0.887334 | 0.538658 | 0.670367 | | 23 | B-Multi-tissue_structure | 228 | 62 | 73 | 0.786207 | 0.757475 | 0.771574 | | 24 | Macro-average | 9159 | 1398 | 2948 | 0.76803 | 0.548396 | 0.639891 | | 25 | Micro-average | 9159 | 1398 | 2948 | 0.867576 | 0.756505 | 0.808242 | ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from AyushPJ) author: John Snow Labs name: roberta_qa_ai_club_inductions_21_nlp_base_squad_v2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `ai-club-inductions-21-nlp-roBERTa-base-squad-v2` is a English model originally trained by `AyushPJ`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_base_squad_v2_en_4.3.0_3.0_1674209021708.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_ai_club_inductions_21_nlp_base_squad_v2_en_4.3.0_3.0_1674209021708.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ai_club_inductions_21_nlp_base_squad_v2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_ai_club_inductions_21_nlp_base_squad_v2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_ai_club_inductions_21_nlp_base_squad_v2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|465.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AyushPJ/ai-club-inductions-21-nlp-roBERTa-base-squad-v2 --- layout: model title: DistilBERT base model (cased) author: John Snow Labs name: distilbert_base_cased date: 2021-05-20 tags: [distilbert, en, english, open_source, embeddings] task: Embeddings language: en nav_key: models edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a distilled version of the [BERT base model](https://huggingface.co/bert-base-cased). It was introduced in [this paper](https://arxiv.org/abs/1910.01108). The code for the distillation process can be found [here](https://github.com/huggingface/transformers/tree/master/examples/research_projects/distillation). This model is cased: it does make a difference between english and English. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_base_cased_en_3.1.0_2.4_1621521790187.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_base_cased_en_3.1.0_2.4_1621521790187.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python embeddings = DistilBertEmbeddings.pretrained("distilbert_base_cased", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) ``` ```scala val embeddings = DistilBertEmbeddings.pretrained("distilbert_base_cased", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) ``` {:.nlu-block} ```python import nlu nlu.load("en.embed.distilbert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_base_cased| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token, sentence]| |Output Labels:|[embeddings]| |Language:|en| |Case sensitive:|true| ## Data Source [https://huggingface.co/distilbert-base-cased](https://huggingface.co/distilbert-base-cased) ## Benchmarking ```bash When fine-tuned on downstream tasks, this model achieves the following results: Glue test results: | Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | |:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| | | 81.5 | 87.8 | 88.2 | 90.4 | 47.2 | 85.5 | 85.6 | 60.6 | ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from MrAnderson) author: John Snow Labs name: bert_qa_base_1024_full_trivia date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-1024-full-trivia` is a English model originally trained by `MrAnderson`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_1024_full_trivia_en_4.0.0_3.0_1657182678581.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_1024_full_trivia_en_4.0.0_3.0_1657182678581.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_1024_full_trivia","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_1024_full_trivia","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_1024_full_trivia| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|409.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/MrAnderson/bert-base-1024-full-trivia --- layout: model title: English asr_english_filipino_wav2vec2_l_xls_r_test_06 TFWav2Vec2ForCTC from Khalsuu author: John Snow Labs name: pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_english_filipino_wav2vec2_l_xls_r_test_06` is a English model originally trained by Khalsuu. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06_en_4.2.0_3.0_1664115059427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06_en_4.2.0_3.0_1664115059427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_english_filipino_wav2vec2_l_xls_r_test_06| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Mapping National Drug Codes (NDC) with Corresponding HCPCS Codes and Descriptions author: John Snow Labs name: ndc_hcpcs_mapper date: 2023-04-13 tags: [en, licensed, clinical, chunk_mapping, ndc, hcpcs, brand_name] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps National Drug Codes (NDC) with their corresponding HCPCS codes and their descriptions. ## Predicted Entities `hcpcs_code`, `hcpcs_description` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ndc_hcpcs_mapper_en_4.4.0_3.0_1681405091593.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ndc_hcpcs_mapper_en_4.4.0_3.0_1681405091593.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ndc_chunk") chunkerMapper = DocMapperModel.pretrained("ndc_hcpcs_mapper", "en", "clinical/models")\ .setInputCols(["ndc_chunk"])\ .setOutputCol("hcpcs")\ .setRels(["hcpcs_code", "hcpcs_description"]) pipeline = Pipeline().setStages([document_assembler, chunkerMapper]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) res = lp.fullAnnotate(["16714-0892-01", "00990-6138-03", "43598-0650-11"]) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text")\ .setOutputCol("ndc_chunk") val chunkerMapper = DocMapperModel .pretrained("ndc_hcpcs_mapper", "en", "clinical/models") .setInputCols(Array("ndc_chunk")) .setOutputCol("mappings") .setRels(Array("hcpcs_code", "hcpcs_description")) val mapper_pipeline = new Pipeline().setStages(Array( document_assembler, chunkerMapper)) val data = Seq(Array("16714-0892-01", "00990-6138-03", "43598-0650-11")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +-------------+----------------------------+-----------------+ |ndc_chunk |mappings |relation | +-------------+----------------------------+-----------------+ |16714-0892-01|J0878 |hcpcs_code | |16714-0892-01|INJECTION, DAPTOMYCIN, 1 MG |hcpcs_description| |00990-6138-03|A4217 |hcpcs_code | |00990-6138-03|STERILE WATER/SALINE, 500 ML|hcpcs_description| |43598-0650-11|J9340 |hcpcs_code | |43598-0650-11|INJECTION, THIOTEPA, 15 MG |hcpcs_description| +-------------+----------------------------+-----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ndc_hcpcs_mapper| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|203.1 KB| --- layout: model title: Sentence Detection in Nepali Text author: John Snow Labs name: sentence_detector_dl date: 2021-08-30 tags: [sentence_detection, ne, open_source] task: Sentence Detection language: ne edition: Spark NLP 3.2.0 spark_version: 3.0 supported: true annotator: SentenceDetectorDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description SentenceDetectorDL (SDDL) is based on a general-purpose neural network model for sentence boundary detection. The task of sentence boundary detection is to identify sentences within a text. Many natural language processing tasks take a sentence as an input unit, such as part-of-speech tagging, dependency parsing, named entity recognition or machine translation. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/9.SentenceDetectorDL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ne_3.2.0_3.0_1630334779183.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sentence_detector_dl_ne_3.2.0_3.0_1630334779183.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencerDL = SentenceDetectorDLModel\ .pretrained("sentence_detector_dl", "ne") \ .setInputCols(["document"]) \ .setOutputCol("sentences") sd_model = LightPipeline(PipelineModel(stages=[documenter, sentencerDL])) sd_model.fullAnnotate("""अंग्रेजी पढ्ने अनुच्छेद को एक महान स्रोत को लागी हेर्दै हुनुहुन्छ? तपाइँ सही ठाउँमा आउनुभएको छ. हालै गरिएको एक अध्ययन अनुसार आजको युवाहरुमा पढ्ने बानी छिटोछिटो घट्दै गएको छ. उनीहरु केहि सेकेन्ड भन्दा बढी को लागी एक दिईएको अंग्रेजी पढ्ने अनुच्छेद मा ध्यान केन्द्रित गर्न सक्दैनन्! साथै, पठन थियो र सबै प्रतियोगी परीक्षा को एक अभिन्न हिस्सा हो। त्यसोभए, तपाइँ तपाइँको पठन कौशल कसरी सुधार गर्नुहुन्छ? यो प्रश्न को जवाफ वास्तव मा अर्को प्रश्न हो: पढ्ने कौशल को उपयोग के हो? पढ्न को मुख्य उद्देश्य 'भावना बनाउन' हो.""") ``` ```scala val documenter = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val model = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "ne") .setInputCols(Array("document")) .setOutputCol("sentence") val pipeline = new Pipeline().setStages(Array(documenter, model)) val data = Seq("अंग्रेजी पढ्ने अनुच्छेद को एक महान स्रोत को लागी हेर्दै हुनुहुन्छ? तपाइँ सही ठाउँमा आउनुभएको छ. हालै गरिएको एक अध्ययन अनुसार आजको युवाहरुमा पढ्ने बानी छिटोछिटो घट्दै गएको छ. उनीहरु केहि सेकेन्ड भन्दा बढी को लागी एक दिईएको अंग्रेजी पढ्ने अनुच्छेद मा ध्यान केन्द्रित गर्न सक्दैनन्! साथै, पठन थियो र सबै प्रतियोगी परीक्षा को एक अभिन्न हिस्सा हो। त्यसोभए, तपाइँ तपाइँको पठन कौशल कसरी सुधार गर्नुहुन्छ? यो प्रश्न को जवाफ वास्तव मा अर्को प्रश्न हो: पढ्ने कौशल को उपयोग के हो? पढ्न को मुख्य उद्देश्य 'भावना बनाउन' हो.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load('ne.sentence_detector').predict("अंग्रेजी पढ्ने अनुच्छेद को एक महान स्रोत को लागी हेर्दै हुनुहुन्छ? तपाइँ सही ठाउँमा आउनुभएको छ. हालै गरिएको एक अध्ययन अनुसार आजको युवाहरुमा पढ्ने बानी छिटोछिटो घट्दै गएको छ. उनीहरु केहि सेकेन्ड भन्दा बढी को लागी एक दिईएको अंग्रेजी पढ्ने अनुच्छेद मा ध्यान केन्द्रित गर्न सक्दैनन्! साथै, पठन थियो र सबै प्रतियोगी परीक्षा को एक अभिन्न हिस्सा हो। त्यसोभए, तपाइँ तपाइँको पठन कौशल कसरी सुधार गर्नुहुन्छ? यो प्रश्न को जवाफ वास्तव मा अर्को प्रश्न हो: पढ्ने कौशल को उपयोग के हो? पढ्न को मुख्य उद्देश्य 'भावना बनाउन' हो.", output_level ='sentence') ```

## Results ```bash +-----------------------------------------------------------------------------------------------------------------------+ |result | +-----------------------------------------------------------------------------------------------------------------------+ |[अंग्रेजी पढ्ने अनुच्छेद को एक महान स्रोत को लागी हेर्दै हुनुहुन्छ?] | |[तपाइँ सही ठाउँमा आउनुभएको छ.] | |[हालै गरिएको एक अध्ययन अनुसार आजको युवाहरुमा पढ्ने बानी छिटोछिटो घट्दै गएको छ.] | |[उनीहरु केहि सेकेन्ड भन्दा बढी को लागी एक दिईएको अंग्रेजी पढ्ने अनुच्छेद मा ध्यान केन्द्रित गर्न सक्दैनन्!] | |[साथै, पठन थियो र सबै प्रतियोगी परीक्षा को एक अभिन्न हिस्सा हो। त्यसोभए, तपाइँ तपाइँको पठन कौशल कसरी सुधार गर्नुहुन्छ?] | |[यो प्रश्न को जवाफ वास्तव मा अर्को प्रश्न हो:] | |[पढ्ने कौशल को उपयोग के हो?] | |[ पढ्न को मुख्य उद्देश्य 'भावना बनाउन' हो।] | +-----------------------------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sentence_detector_dl| |Compatibility:|Spark NLP 3.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[sentences]| |Language:|ne| ## Benchmarking ```bash label Accuracy Recall Prec F1 0 0.98 1.00 0.96 0.98 ``` --- layout: model title: Ganda asr_wav2vec2_luganda_by_indonesian_nlp TFWav2Vec2ForCTC from indonesian-nlp author: John Snow Labs name: asr_wav2vec2_luganda_by_indonesian_nlp date: 2022-09-24 tags: [wav2vec2, lg, audio, open_source, asr] task: Automatic Speech Recognition language: lg edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_luganda_by_indonesian_nlp` is a Ganda model originally trained by indonesian-nlp. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_luganda_by_indonesian_nlp_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_indonesian_nlp_lg_4.2.0_3.0_1664036258895.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_luganda_by_indonesian_nlp_lg_4.2.0_3.0_1664036258895.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_luganda_by_indonesian_nlp", "lg")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_luganda_by_indonesian_nlp", "lg") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_luganda_by_indonesian_nlp| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|lg| |Size:|1.2 GB| --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-0` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0_en_4.3.0_3.0_1674215810378.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0_en_4.3.0_3.0_1674215810378.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_few_shot_k_64_finetuned_squad_seed_0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|419.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-0 --- layout: model title: Part of Speech for Amharic author: John Snow Labs name: pos_ud_att date: 2021-03-09 tags: [part_of_speech, open_source, amharic, pos_ud_att, am] task: Part of Speech Tagging language: am edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - NOUN - DET - PART - VERB - PRON - PUNCT - AUX - PROPN - ADP - SCONJ - ADV - CCONJ - ADJ - INTJ - NUM {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_att_am_3.0.0_3.0_1615292425835.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_att_am_3.0.0_3.0_1615292425835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_att", "am") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Hello from John Snow Labs!']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_att", "am") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Hello from John Snow Labs!").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs!""] token_df = nlu.load('am.pos').predict(text) token_df ```

## Results ```bash token pos 0 Hello NOUN 1 from NOUN 2 John NOUN 3 Snow VERB 4 Labs VERB 5 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_att| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|am| --- layout: model title: Swedish asr_wav2vec2_large_voxrex_swedish_4gram TFWav2Vec2ForCTC from viktor-enzell author: John Snow Labs name: asr_wav2vec2_large_voxrex_swedish_4gram date: 2022-09-25 tags: [wav2vec2, sv, audio, open_source, asr] task: Automatic Speech Recognition language: sv edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_voxrex_swedish_4gram` is a Swedish model originally trained by viktor-enzell. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_voxrex_swedish_4gram_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_voxrex_swedish_4gram_sv_4.2.0_3.0_1664113748205.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_voxrex_swedish_4gram_sv_4.2.0_3.0_1664113748205.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_voxrex_swedish_4gram", "sv")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_voxrex_swedish_4gram", "sv") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_voxrex_swedish_4gram| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|sv| |Size:|757.4 MB| --- layout: model title: Detect Clinical Entities (jsl_ner_wip_greedy_biobert) author: John Snow Labs name: jsl_ner_wip_greedy_biobert date: 2021-07-26 tags: [ner, licensed, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.1.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. ## Predicted Entities `Test_Result`, `Relationship_Status`, `RelativeDate`, `Blood_Pressure`, `Triglycerides`, `Smoking`, `Pregnancy`, `Medical_History_Header`, `LDL`, `Hypertension`, `Hyperlipidemia`, `Frequency`, `BMI`, `Internal_organ_or_component`, `Allergen`, `Fetus_NewBorn`, `Substance_Quantity`, `Time`, `Temperature`, `Procedure`, `Strength`, `Treatment`, `HDL`, `Alcohol`, `Birth_Entity`, `Diet`, `Weight`, `Oxygen_Therapy`, `Injury_or_Poisoning`, `Section_Header`, `Obesity`, `EKG_Findings`, `Gender`, `Height`, `Social_History_Header`, `Diabetes`, `Route`, `Race_Ethnicity`, `Substance`, `Drug`, `External_body_part_or_region`, `RelativeTime`, `Admission_Discharge`, `Psychological_Condition`, `Total_Cholesterol`, `Labour_Delivery`, `Imaging_Technique`, `Date`, `Form`, `Overweight`, `Cerebrovascular_Disease`, `Vital_Signs_Header`, `Oncological`, `ImagingFindings`, `Communicable_Disease`, `Duration`, `Vaccine`, `Kidney_Disease`, `O2_Saturation`, `Heart_Disease`, `Employment`, `Sexually_Active_or_Sexual_Orientation`, `Test`, `Disease_Syndrome_Disorder`, `Respiration`, `Direction`, `Medical_Device`, `Clinical_Dept`, `Modifier`, `Symptom`, `Pulse`, `Age`, `Death_Entity`, `Dosage`, `Family_History_Header`, `VS_Finding` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_biobert_en_3.1.3_3.0_1627304288213.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/jsl_ner_wip_greedy_biobert_en_3.1.3_3.0_1627304288213.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunk | entity | |---:|:-----------------------------------------------|:-----------------------------| | 0 | 21-day-old | Age | | 1 | Caucasian | Race_Ethnicity | | 2 | male | Gender | | 3 | for 2 days | Duration | | 4 | congestion | Symptom | | 5 | mom | Gender | | 6 | suctioning yellow discharge | Symptom | | 7 | nares | External_body_part_or_region | | 8 | she | Gender | | 9 | mild problems with his breathing while feeding | Symptom | | 10 | perioral cyanosis | Symptom | | 11 | retractions | Symptom | | 12 | One day ago | RelativeDate | | 13 | mom | Gender | | 14 | tactile temperature | Symptom | | 15 | Tylenol | Drug | | 16 | Baby | Age | | 17 | decreased p.o. intake | Symptom | | 18 | His | Gender | | 19 | breast-feeding | External_body_part_or_region | | 20 | q.2h | Frequency | | 21 | to 5 to 10 minutes | Duration | | 22 | his | Gender | | 23 | respiratory congestion | Symptom | | 24 | He | Gender | | 25 | tired | Symptom | | 26 | fussy | Symptom | | 27 | over the past 2 days | RelativeDate | | 28 | albuterol | Drug | | 29 | ER | Clinical_Dept | | 30 | His | Gender | | 31 | urine output has also decreased | Symptom | | 32 | he | Gender | | 33 | per 24 hours | Frequency | | 34 | he | Gender | | 35 | per 24 hours | Frequency | | 36 | Mom | Gender | | 37 | diarrhea | Symptom | | 38 | His | Gender | | 39 | bowel | Internal_organ_or_component | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|jsl_ner_wip_greedy_biobert| |Compatibility:|Healthcare NLP 3.1.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash label tp fp fn prec rec f1 B-Oxygen_Therapy 47 11 10 0.8103448 0.8245614 0.81739134 B-Cerebrovascular_Disease 43 20 21 0.6825397 0.671875 0.6771653 B-Triglycerides 5 0 0 1.0 1.0 1.0 I-Cerebrovascular_Disease 25 12 27 0.6756757 0.48076922 0.56179774 B-Medical_Device 2704 531 364 0.8358578 0.88135594 0.85800415 B-Labour_Delivery 43 16 29 0.7288136 0.5972222 0.6564886 I-Vaccine 5 0 5 1.0 0.5 0.6666667 I-Obesity 6 4 1 0.6 0.85714287 0.70588243 I-Smoking 3 1 2 0.75 0.6 0.6666667 B-RelativeTime 67 36 51 0.65048546 0.5677966 0.60633487 B-Imaging_Technique 33 12 19 0.73333335 0.63461536 0.68041235 B-Heart_Disease 285 55 68 0.8382353 0.8073654 0.82251084 B-Procedure 1876 303 384 0.8609454 0.8300885 0.84523547 I-RelativeTime 105 43 53 0.7094595 0.664557 0.6862745 B-Drug 1803 299 265 0.8577545 0.87185687 0.8647482 B-Obesity 29 9 5 0.7631579 0.85294116 0.8055555 I-RelativeDate 617 167 107 0.7869898 0.8522099 0.8183024 B-O2_Saturation 27 8 6 0.7714286 0.8181818 0.7941177 B-Direction 2856 390 326 0.8798521 0.89754874 0.88861233 I-Alcohol 4 4 4 0.5 0.5 0.5 I-Oxygen_Therapy 25 7 6 0.78125 0.8064516 0.79365087 B-Diet 23 14 32 0.6216216 0.4181818 0.5 B-Dosage 35 26 29 0.57377046 0.546875 0.55999994 B-Injury_or_Poisoning 308 52 83 0.85555553 0.7877238 0.82023966 B-Hypertension 80 9 2 0.8988764 0.9756098 0.9356726 I-Test_Result 124 73 156 0.6294416 0.44285715 0.5199161 B-Alcohol 54 11 12 0.830 0.8181818 0.8244275 B-Height 14 5 5 0.7368421 0.7368421 0.7368421 I-Substance 18 8 8 0.6923077 0.6923077 0.6923077 B-RelativeDate 372 109 93 0.7733888 0.8 0.78646934 B-Admission_Discharge 218 22 14 0.90833336 0.9396552 0.9237288 B-Date 345 24 26 0.93495935 0.9299191 0.9324324 B-Kidney_Disease 63 10 20 0.8630137 0.7590361 0.8076923 I-Strength 22 17 13 0.5641026 0.62857145 0.59459466 I-Injury_or_Poisoning 301 93 98 0.7639594 0.75438595 0.75914246 I-Time 28 11 17 0.71794873 0.62222224 0.6666667 B-Substance 48 11 10 0.8135593 0.82758623 0.8205129 B-Total_Cholesterol 6 3 0 0.6666667 1.0 0.8 I-Vital_Signs_Header 276 28 8 0.90789473 0.97183096 0.93877554 I-Internal_organ_or_component 2907 518 490 0.8487591 0.8557551 0.8522427 B-Hyperlipidemia 28 3 0 0.9032258 1.0 0.9491525 B-Overweight 3 0 3 1.0 0.5 0.6666667 I-Sexually_Active_or_Sexual_Orientation 2 0 3 1.0 0.4 0.5714286 B-Sexually_Active_or_Sexual_Orientation 2 0 2 1.0 0.5 0.6666667 I-Fetus_NewBorn 50 38 58 0.5681818 0.46296296 0.5102041 B-BMI 6 0 1 1.0 0.85714287 0.9230769 B-ImagingFindings 52 41 61 0.5591398 0.460177 0.5048544 B-Test_Result 714 135 212 0.8409894 0.7710583 0.8045071 B-Section_Header 2140 79 65 0.9643984 0.97052157 0.96745026 I-Treatment 85 21 29 0.8018868 0.74561405 0.7727273 B-Clinical_Dept 638 82 77 0.88611114 0.8923077 0.88919866 I-Kidney_Disease 114 7 18 0.94214875 0.8636364 0.90118575 I-Pulse 189 27 42 0.875 0.8181818 0.84563756 B-Test 1589 320 315 0.83237296 0.83455884 0.83346444 B-Weight 54 12 13 0.8181818 0.80597013 0.81203 I-Respiration 114 4 17 0.9661017 0.870229 0.91566265 I-EKG_Findings 68 34 52 0.6666667 0.56666666 0.6126126 I-Section_Header 3828 168 77 0.957958 0.9802817 0.9689913 B-Strength 27 13 23 0.675 0.54 0.6 I-Social_History_Header 137 4 4 0.9716312 0.9716312 0.9716312 B-Vital_Signs_Header 183 18 7 0.9104478 0.9631579 0.9360614 B-Death_Entity 28 9 6 0.7567568 0.8235294 0.7887324 B-Modifier 302 90 282 0.77040815 0.5171233 0.6188525 B-Blood_Pressure 93 14 21 0.86915886 0.81578946 0.84162897 I-O2_Saturation 49 19 23 0.7205882 0.6805556 0.7 B-Frequency 437 77 68 0.8501946 0.86534655 0.8577036 I-Triglycerides 5 0 0 1.0 1.0 1.0 I-Duration 513 254 47 0.66883963 0.9160714 0.77317256 I-Diabetes 50 4 6 0.9259259 0.89285713 0.90909094 B-Race_Ethnicity 78 3 2 0.962963 0.975 0.9689441 I-Gender 114 2 17 0.98275864 0.870229 0.9230769 I-Height 43 13 10 0.76785713 0.8113208 0.78899086 B-Communicable_Disease 10 5 9 0.6666667 0.5263158 0.5882354 I-Family_History_Header 134 1 0 0.9925926 1.0 0.9962825 B-LDL 2 2 2 0.5 0.5 0.5 I-Race_Ethnicity 6 0 0 1.0 1.0 1.0 B-Psychological_Condition 103 21 17 0.83064514 0.85833335 0.84426236 I-Age 116 14 50 0.8923077 0.6987952 0.78378385 B-EKG_Findings 33 18 32 0.64705884 0.50769234 0.56896555 B-Employment 168 29 44 0.8527919 0.7924528 0.8215159 I-Oncological 358 38 17 0.9040404 0.9546667 0.9286641 B-Time 27 7 18 0.7941176 0.6 0.68354434 B-Treatment 93 31 41 0.75 0.69402987 0.7209303 B-Temperature 69 5 8 0.9324324 0.8961039 0.9139073 I-Procedure 2437 379 501 0.86541194 0.8294758 0.84706295 B-Relationship_Status 30 3 1 0.90909094 0.9677419 0.9375 B-Pregnancy 56 17 30 0.7671233 0.6511628 0.7044025 I-Route 8 4 7 0.6666667 0.53333336 0.59259266 I-Medical_History_Header 151 4 15 0.9741936 0.9096386 0.94080997 I-Imaging_Technique 25 5 20 0.8333333 0.5555556 0.66666675 B-Smoking 74 6 4 0.925 0.94871795 0.93670887 I-Labour_Delivery 36 8 18 0.8181818 0.6666667 0.7346939 I-Death_Entity 3 0 2 1.0 0.6 0.75 B-Diabetes 77 9 5 0.89534885 0.9390244 0.9166666 B-Gender 4479 82 111 0.9820215 0.97581697 0.9789094 B-Vaccine 6 1 9 0.85714287 0.4 0.54545456 I-Heart_Disease 393 61 89 0.8656388 0.8153527 0.8397436 I-Dosage 31 27 22 0.5344828 0.5849057 0.5585586 B-Social_History_Header 78 2 3 0.975 0.962963 0.9689441 B-External_body_part_or_region 1640 402 311 0.8031342 0.8405946 0.8214376 I-Clinical_Dept 546 59 47 0.90247935 0.920742 0.91151917 I-Test 1195 320 402 0.7887789 0.748278 0.7679949 I-Frequency 340 97 120 0.77803206 0.73913044 0.75808245 B-Age 454 35 57 0.9284254 0.888454 0.908 B-Pulse 90 11 17 0.8910891 0.8411215 0.8653846 I-Symptom 4265 2050 1232 0.6753761 0.7758778 0.72214705 I-Pregnancy 39 28 42 0.58208954 0.4814815 0.527027 I-LDL 5 0 4 1.0 0.5555556 0.71428573 I-Diet 33 14 25 0.70212764 0.5689655 0.6285714 I-Blood_Pressure 198 54 27 0.78571427 0.88 0.83018863 I-ImagingFindings 136 99 85 0.57872343 0.61538464 0.5964913 I-Date 203 13 10 0.9398148 0.9530516 0.946387 B-Route 84 23 47 0.78504676 0.64122134 0.7058824 B-Duration 204 110 26 0.6496815 0.8869565 0.74999994 B-Medical_History_Header 56 1 7 0.98245615 0.8888889 0.93333334 B-Respiration 55 4 6 0.9322034 0.90163934 0.9166667 I-External_body_part_or_region 314 105 167 0.74940336 0.65280664 0.6977778 I-BMI 15 0 1 1.0 0.9375 0.9677419 B-Internal_organ_or_component 4349 886 761 0.8307545 0.8510763 0.8407926 I-Weight 150 22 23 0.872093 0.867052 0.8695652 B-Disease_Syndrome_Disorder 1698 375 358 0.81910276 0.82587546 0.8224752 B-Symptom 4358 1002 932 0.8130597 0.8238185 0.8184037 B-VS_Finding 138 36 37 0.79310346 0.7885714 0.79083097 I-Disease_Syndrome_Disorder 1723 372 451 0.82243437 0.7925483 0.8072148 I-Drug 3282 838 493 0.79660195 0.86940396 0.8314123 I-Medical_Device 1864 418 242 0.81682736 0.88509023 0.84958977 B-Oncological 278 22 22 0.9266667 0.9266667 0.9266667 I-Temperature 111 8 6 0.9327731 0.94871795 0.94067794 I-Employment 92 27 19 0.77310926 0.8288288 0.8 I-Psychological_Condition 32 7 19 0.82051283 0.627451 0.7111111 B-Family_History_Header 68 0 0 1.0 1.0 1.0 I-Direction 311 91 144 0.7736318 0.6835165 0.72578764 Macro-average 65035 12855 11898 0.761429 0.70630085 0.7328297 Micro-average 65035 12855 11898 0.83495957 0.845346 0.8401207 ``` --- layout: model title: Translate Salishan languages to English Pipeline author: John Snow Labs name: translate_sal_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, sal, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `sal` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_sal_en_xx_2.7.0_2.4_1609686923332.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_sal_en_xx_2.7.0_2.4_1609686923332.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_sal_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_sal_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.sal.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_sal_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Proposal Classification author: John Snow Labs name: legclf_proposal_topic date: 2023-02-17 tags: [en, legal, classification, proposal, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Given a proposal on a socially important issue, this model classifies it according to its topic. ## Predicted Entities `Democracy`, `Digital`, `EU_In_The_World`, `Economy`, `Education`, `Green_Deal`, `Health`, `Migration`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_proposal_topic_en_1.0.0_3.0_1676594573703.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_proposal_topic_en_1.0.0_3.0_1676594573703.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_embeddings = nlp.UniversalSentenceEncoder.pretrained()\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") classifier = legal.ClassifierDLModel.pretrained("legclf_proposal_topic", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("class") clf_pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_embeddings, classifier ]) empty_df = spark.createDataFrame([['']]).toDF("text") model = clf_pipeline.fit(empty_df) text = ["""In order to involve young people in the European Union, they need to understand the role, importance, and impact of the European Union on their lives and how they can contribute to the EU. I believe that many Europeans do not know the values of Europe, how they can contribute to the EU, etc. To do this, it was necessary to create an education program on the European Union that could cut across all countries, including a discipline on the EU, visits by young people to the European institutions, and a 'channel of communication' between young people and the EU. The same could be done for older people in senior universities."""] data = spark.createDataFrame([text]).toDF("text") result = model.transform(data) ```

## Results ```bash +-----------+ | result| +-----------+ |[Education]| +-----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_proposal_topic| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Training dataset available [here](https://touche.webis.de/clef23/touche23-web/multilingual-stance-classification.html#data) ## Benchmarking ```bash label precision recall f1-score support Democracy 0.86 0.90 0.88 62 Digital 0.85 0.80 0.82 35 EU_In_The_World 0.78 0.72 0.75 39 Economy 0.82 0.77 0.80 43 Education 0.89 0.87 0.88 46 Green_Deal 0.85 0.92 0.88 49 Health 0.87 0.95 0.91 21 Migration 0.86 0.89 0.87 27 Other 1.00 0.97 0.98 32 accuracy - - 0.86 354 macro-avg 0.86 0.87 0.86 354 weighted-avg 0.86 0.86 0.86 354 ``` --- layout: model title: Marathi Bert Embeddings author: John Snow Labs name: bert_embeddings_marathi_bert date: 2022-04-11 tags: [bert, embeddings, mr, open_source] task: Embeddings language: mr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `marathi-bert` is a Marathi model orginally trained by `l3cube-pune`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_marathi_bert_mr_3.4.2_3.0_1649675042393.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_marathi_bert_mr_3.4.2_3.0_1649675042393.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_marathi_bert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|mr| |Size:|668.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/l3cube-pune/marathi-bert - https://github.com/l3cube-pune/MarathiNLP - https://arxiv.org/abs/2202.01159 --- layout: model title: German Electra Embeddings (from stefan-it) author: John Snow Labs name: electra_embeddings_electra_base_gc4_64k_700000_cased_generator date: 2022-05-17 tags: [de, open_source, electra, embeddings] task: Embeddings language: de edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-gc4-64k-700000-cased-generator` is a German model orginally trained by `stefan-it`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_700000_cased_generator_de_3.4.4_3.0_1652786477097.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_gc4_64k_700000_cased_generator_de_3.4.4_3.0_1652786477097.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_gc4_64k_700000_cased_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|de| |Size:|222.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/stefan-it/electra-base-gc4-64k-700000-cased-generator - https://german-nlp-group.github.io/projects/gc4-corpus.html - https://faculty.washington.edu/ebender/papers/Stochastic_Parrots.pdf --- layout: model title: NER Pipeline for 6 Scandinavian Languages author: John Snow Labs name: bert_token_classifier_scandi_ner_pipeline date: 2022-02-15 tags: [danish, norwegian, swedish, icelandic, faroese, bert, xx, open_source] task: Named Entity Recognition language: xx edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on [bert_token_classifier_scandi_ner](https://nlp.johnsnowlabs.com/2021/12/09/bert_token_classifier_scandi_ner_xx.html) model which is imported from `HuggingFace`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_SCANDINAVIAN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_scandi_ner_pipeline_xx_3.4.0_3.0_1644927539839.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_scandi_ner_pipeline_xx_3.4.0_3.0_1644927539839.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------+---------+ |chunk |ner_label| +-------------------+---------+ |Hans |PER | |Statens Universitet|ORG | |København |LOC | |københavner |MISC | +-------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_scandi_ner_pipeline| |Type:|pipeline| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Language:|xx| |Size:|666.9 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - BertForTokenClassification - NerConverter - Finisher --- layout: model title: Pipeline to Detect diseases in text (large) author: John Snow Labs name: ner_diseases_large_pipeline date: 2023-03-14 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_diseases_large](https://nlp.johnsnowlabs.com/2021/04/01/ner_diseases_large_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_diseases_large_pipeline_en_4.3.0_3.2_1678836753073.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_diseases_large_pipeline_en_4.3.0_3.2_1678836753073.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_diseases_large_pipeline", "en", "clinical/models") text = '''Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_diseases_large_pipeline", "en", "clinical/models") val text = "Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.diseases_large.pipeline").predict("""Detection of various other intracellular signaling proteins is also described. Genetic characterization of transactivation of the human T-cell leukemia virus type 1 promoter: Binding of Tax to Tax-responsive element 1 is mediated by the cyclic AMP-responsive members of the CREB/ATF family of transcription factors. To achieve a better understanding of the mechanism of transactivation by Tax of human T-cell leukemia virus type 1 Tax-responsive element 1 (TRE-1), we developed a genetic approach with Saccharomyces cerevisiae. We constructed a yeast reporter strain containing the lacZ gene under the control of the CYC1 promoter associated with three copies of TRE-1. Expression of either the cyclic AMP response element-binding protein (CREB) or CREB fused to the GAL4 activation domain (GAD) in this strain did not modify the expression of the reporter gene. Tax alone was also inactive.""") ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:----------------|--------:|------:|:------------|-------------:| | 0 | T-cell leukemia | 136 | 150 | Disease | 0.93585 | | 1 | T-cell leukemia | 402 | 416 | Disease | 0.9567 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_diseases_large_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from ChutianTao) author: John Snow Labs name: distilbert_qa_base_uncased_finetuned_squad_1 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad-1` is a English model originally trained by `ChutianTao`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad_1_en_4.3.0_3.0_1672773434592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_finetuned_squad_1_en_4.3.0_3.0_1672773434592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_finetuned_squad_1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_finetuned_squad_1| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/ChutianTao/distilbert-base-uncased-finetuned-squad-1 --- layout: model title: Sentence Entity Resolver for ICD10-CM (Augmented) author: John Snow Labs name: sbiobertresolve_icd10cm_augmented date: 2023-05-20 tags: [icd10cm, entity_resolution, clinical, en, licensed] task: Entity Resolution language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to ICD10-CM codes using `sbiobert_base_cased_mli` Sentence Bert Embeddings. Also, it has been augmented with synonyms for making it more accurate and returns the official resolution text within the brackets. ## Predicted Entities `ICD10CM Codes` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_4.4.2_3.0_1684591054927.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_icd10cm_augmented_en_4.4.2_3.0_1684591054927.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------+-------+------------+-------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------+ | ner_chunk| entity|icd10cm_code| resolutions | all_codes| +-------------------------------------+-------+------------+-------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------+ | gestational diabetes mellitus|PROBLEM| O24.4|gestational diabetes mellitus [gestational diabetes mellitus], gestational diabetes mellitus [gestational... |O24.4, O24.41, O24.43, Z86.32, Z87.5, O24.31, O24.11, O24.1, O24.81...| |subsequent type two diabetes mellitus|PROBLEM| O24.11|pre-existing type 2 diabetes mellitus [pre-existing type 2 diabetes mellitus], disorder associated with t... |O24.11, E11.8, E11, E13.9, E11.9, E11.3, E11.44, Z86.3, Z86.39, E11...| | obesity|PROBLEM| E66.9|obesity [obesity], abdominal obesity [abdominal obesity], obese [obese], central obesity [central obesity... |E66.9, E66.8, Z68.41, Q13.0, E66, E66.01, Z86.39, E34.9, H35.50, Z8...| | a body mass index|PROBLEM| Z68.41|finding of body mass index [finding of body mass index], observation of body mass index [observation of b... |Z68.41, E66.9, R22.9, Z68.1, R22.3, R22.1, Z68, R22.2, R22.0, R41.8...| | polyuria|PROBLEM| R35|polyuria [polyuria], nocturnal polyuria [nocturnal polyuria], polyuric state [polyuric state], polyuric ... |R35, R35.81, R35.8, E23.2, R35.89, R31, R35.0, R82.99, N40.1, E72.3...| | polydipsia|PROBLEM| R63.1|polydipsia [polydipsia], psychogenic polydipsia [psychogenic polydipsia], primary polydipsia [primary po... |R63.1, F63.89, E23.2, F63.9, O40, G47.5, M79.89, R63.2, R06.1, H53....| | poor appetite|PROBLEM| R63.0|poor appetite [poor appetite], poor feeding [poor feeding], bad taste in mouth [bad taste in mouth], unp... |R63.0, P92.9, R43.8, R43.2, E86, R19.6, F52.0, Z72.4, R06.89, Z76.8...| | vomiting|PROBLEM| R11.1|vomiting [vomiting], intermittent vomiting [intermittent vomiting], vomiting symptoms [vomiting symptom... |R11.1, R11, R11.10, G43.A1, P92.1, P92.09, G43.A, R11.13, R11.0 | | a respiratory tract infection|PROBLEM| J98.8|respiratory tract infection [respiratory tract infection], upper respiratory tract infection [upper respi... |J98.8, J06.9, A49.9, J22, J20.9, Z59.3, T17, J04.10, Z13.83, J18.9 ...| +-------------------------------------+-------+------------+----------------------------------------------------------------------+--------------------------------------+----------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_icd10cm_augmented| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[icd10cm_code]| |Language:|en| |Size:|1.5 GB| |Case sensitive:|false| ## References Trained on ICD10CM 2023 Codes dataset: https://www.cdc.gov/nchs/icd/icd10cm.htm --- layout: model title: Legal Organisation Of Work And Working Conditions Document Classifier (EURLEX) author: John Snow Labs name: legclf_organisation_of_work_and_working_conditions_bert date: 2023-03-06 tags: [en, legal, classification, clauses, organisation_of_work_and_working_conditions, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_organisation_of_work_and_working_conditions_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Organisation_of_Work_and_Working_Conditions or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Organisation_of_Work_and_Working_Conditions`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_organisation_of_work_and_working_conditions_bert_en_1.0.0_3.0_1678111659189.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_organisation_of_work_and_working_conditions_bert_en_1.0.0_3.0_1678111659189.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_organisation_of_work_and_working_conditions_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Organisation_of_Work_and_Working_Conditions]| |[Other]| |[Other]| |[Organisation_of_Work_and_Working_Conditions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_organisation_of_work_and_working_conditions_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Organisation_of_Work_and_Working_Conditions 1.00 0.96 0.98 23 Other 0.96 1.00 0.98 24 accuracy - - 0.98 47 macro-avg 0.98 0.98 0.98 47 weighted-avg 0.98 0.98 0.98 47 ``` --- layout: model title: Croatian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_croatian_legal date: 2023-02-16 tags: [hr, croatian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: hr edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-croatian-roberta-base` is a Croatian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_croatian_legal_hr_4.2.4_3.0_1676576201975.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_croatian_legal_hr_4.2.4_3.0_1676576201975.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_croatian_legal", "hr")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_croatian_legal", "hr") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_croatian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|hr| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-croatian-roberta-base --- layout: model title: English T5ForConditionalGeneration Base Cased model (from tennessejoyce) author: John Snow Labs name: t5_titlewave_base date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `titlewave-t5-base` is a English model originally trained by `tennessejoyce`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_titlewave_base_en_4.3.0_3.0_1675157398835.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_titlewave_base_en_4.3.0_3.0_1675157398835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_titlewave_base","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_titlewave_base","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_titlewave_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|917.2 MB| ## References - https://huggingface.co/tennessejoyce/titlewave-t5-base - https://github.com/tennessejoyce/TitleWave - https://github.com/tennessejoyce/TitleWave - https://archive.org/details/stackexchange - https://github.com/tennessejoyce/TitleWave/blob/master/model_training/test_summarizer.ipynb --- layout: model title: Hindi asr_CDAC_hindispeechrecognition TFWav2Vec2ForCTC from nalini2799 author: John Snow Labs name: asr_CDAC_hindispeechrecognition date: 2022-09-26 tags: [wav2vec2, hi, audio, open_source, asr] task: Automatic Speech Recognition language: hi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_CDAC_hindispeechrecognition` is a Hindi model originally trained by nalini2799. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_CDAC_hindispeechrecognition_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_CDAC_hindispeechrecognition_hi_4.2.0_3.0_1664188438122.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_CDAC_hindispeechrecognition_hi_4.2.0_3.0_1664188438122.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_CDAC_hindispeechrecognition", "hi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_CDAC_hindispeechrecognition", "hi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_CDAC_hindispeechrecognition| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|hi| |Size:|1.2 GB| --- layout: model title: Chinese Bert Embeddings (from shibing624) author: John Snow Labs name: bert_embeddings_macbert4csc_base_chinese date: 2022-04-11 tags: [bert, embeddings, zh, open_source] task: Embeddings language: zh edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `macbert4csc-base-chinese` is a Chinese model orginally trained by `shibing624`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_macbert4csc_base_chinese_zh_3.4.2_3.0_1649669240219.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_macbert4csc_base_chinese_zh_3.4.2_3.0_1649669240219.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_macbert4csc_base_chinese| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|384.0 MB| |Case sensitive:|true| ## References - https://huggingface.co/shibing624/macbert4csc-base-chinese - https://github.com/shibing624/pycorrector - https://pan.baidu.com/s/1BV5tr9eONZCI0wERFvr0gQ - http://nlp.ee.ncu.edu.tw/resource/csc.html - https://github.com/wdimmy/Automatic-Corpus-Generation/blob/master/corpus/train.sgml - https://github.com/shibing624/pycorrector/tree/master/pycorrector/macbert - https://arxiv.org/abs/2004.13922 --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_4_h_128 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-4_H-128` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_128_zh_4.2.4_3.0_1670325916540.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_4_h_128_zh_4.2.4_3.0_1670325916540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_128","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_4_h_128","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_4_h_128| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|13.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-4_H-128 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Romanian T5ForConditionalGeneration Small Cased model (from BlackKakapo) author: John Snow Labs name: t5_small_paraphrase_v2 date: 2023-01-31 tags: [ro, open_source, t5, tensorflow] task: Text Generation language: ro edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-small-paraphrase-ro-v2` is a Romanian model originally trained by `BlackKakapo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_small_paraphrase_v2_ro_4.3.0_3.0_1675155540424.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_small_paraphrase_v2_ro_4.3.0_3.0_1675155540424.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_small_paraphrase_v2","ro") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_small_paraphrase_v2","ro") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_small_paraphrase_v2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|ro| |Size:|288.8 MB| ## References - https://huggingface.co/BlackKakapo/t5-small-paraphrase-ro-v2 - https://img.shields.io/badge/V.2-17.08.2022-brightgreen --- layout: model title: Persian Named Entity Recognition (from HooshvareLab) author: John Snow Labs name: bert_ner_bert_fa_zwnj_base_ner date: 2022-05-09 tags: [bert, ner, token_classification, fa, open_source] task: Named Entity Recognition language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-fa-zwnj-base-ner` is a Persian model orginally trained by `HooshvareLab`. ## Predicted Entities `LOC`, `PRO`, `MON`, `TIM`, `PER`, `DAT`, `FAC`, `EVE`, `PCT`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_bert_fa_zwnj_base_ner_fa_3.4.2_3.0_1652099703343.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_bert_fa_zwnj_base_ner_fa_3.4.2_3.0_1652099703343.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_bert_fa_zwnj_base_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|fa| |Size:|442.2 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/HooshvareLab/bert-fa-zwnj-base-ner - https://github.com/HaniehP/PersianNER - http://nsurl.org/2019-2/tasks/task-7-named-entity-recognition-ner-for-farsi/ - https://elisa-ie.github.io/wikiann/ - https://github.com/hooshvare/parsner/issues --- layout: model title: Financial News Summarization (Medium) author: John Snow Labs name: finsum_news_md date: 2022-11-24 tags: [en, licensed] task: Summarization language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Financial news Summarizer, finetuned with a financial dataset (about 10K news). ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finsum_news_md_en_1.0.0_3.0_1669312993098.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finsum_news_md_en_1.0.0_3.0_1669312993098.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = nlp.T5Transformer() \ .pretrained("finsum_news_md" ,"en", "finance/models") \ .setTask("summarization") \ # or summarize: .setInputCols(["documents"]) \ .setMaxOutputLength(512) \ .setOutputCol("summaries") data_df = spark.createDataFrame([["Deere Grows Sales 37% as Shipments Rise. Farm equipment supplier forecasts higher sales in year ahead, lifted by price increases and infrastructure investments. Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more of its farm and construction equipment. The Moline, Ill.-based company, the largest supplier of farm equipment in the U.S., said demand held up as it raised prices on farm equipment, and forecast sales gains in the year ahead. Chief Executive John May cited strong demand and increased investment in infrastructure projects as the Biden administration ramps up spending. Elevated crop prices have kept farmers interested in new machinery even as their own production expenses increase."]]).toDF("text") pipeline = nlp.Pipeline().setStages([document_assembler, t5]) results = pipeline.fit(data_df).transform(data_df) results.select("summaries.result").show(truncate=False) ```

## Results ```bash Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more farm and construction equipment. Deere & Co. said its fiscal fourth-quarter sales surged 37% as supply constraints eased and the company shipped more farm and construction equipment. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finsum_news_md| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[summaries]| |Language:|en| |Size:|925.1 MB| ## References John Snow Labs in-house summarized articles. --- layout: model title: English image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3 ViTForImageClassification from AykeeSalazar author: John Snow Labs name: image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3` is a English model originally trained by AykeeSalazar. ## Predicted Entities `nonViolation`, `publicDrinking`, `publicSmoking` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3_en_4.1.0_3.0_1660168819647.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3_en_4.1.0_3.0_1660168819647.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_vc_bantai__withoutAMBI_adunest_v3| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: Igbo Named Entity Recognition (from Davlan) author: John Snow Labs name: distilbert_ner_distilbert_base_multilingual_cased_masakhaner date: 2022-05-16 tags: [distilbert, ner, token_classification, ig, open_source] task: Named Entity Recognition language: ig edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-multilingual-cased-masakhaner` is a Igbo model orginally trained by `Davlan`. ## Predicted Entities `DATE`, `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_multilingual_cased_masakhaner_ig_3.4.2_3.0_1652721792853.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_ner_distilbert_base_multilingual_cased_masakhaner_ig_3.4.2_3.0_1652721792853.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_ner_distilbert_base_multilingual_cased_masakhaner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ig| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Davlan/distilbert-base-multilingual-cased-masakhaner - https://github.com/masakhane-io/masakhane-ner - https://github.com/masakhane-io/masakhane-ner - https://arxiv.org/abs/2103.11811 --- layout: model title: Named Entity Recognition (NER) Model in Finnish (GloVe 6B 300) author: John Snow Labs name: finnish_ner_6B_300 date: 2020-09-01 task: Named Entity Recognition language: fi edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [ner, fi, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Finnish NER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. The model is trained with GloVe 6B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Product-`PRO`, Date-`DATE`, Event-`EVENT`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/PP_EXPLAIN_DOCUMENT_FI/){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/PP_EXPLAIN_DOCUMENT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/finnish_ner_6B_300_fi_2.6.0_2.4_1598965807718.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/finnish_ner_6B_300_fi_2.6.0_2.4_1598965807718.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |William Henry Gates III|PER | |lokakuuta 1955 |DATE | |ulkoministeriöitä |ORG | |Microsoft |ORG | |Microsoft Corporationin|PRO | |Microsoft |ORG | |Gates |PER | |2014 |DATE | |1970'erne 1980 |DATE | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | |Gates |PER | |tammikuu 2000 |DATE | |Gates |PER | |Denne |PER | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finnish_ner_6B_300| |Type:|ner| |Compatibility:| Spark NLP 2.6.0+| |Edition:|Official| |License:|Open Source| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|fi| |Case sensitive:|false| {:.h2_title} ## Data Source The detailed information can be found from [https://www.aclweb.org/anthology/2020.lrec-1.567.pdf](https://www.aclweb.org/anthology/2020.lrec-1.567.pdf) --- layout: model title: Russian DistilBertForQuestionAnswering model (from AndrewChar) author: John Snow Labs name: distilbert_qa_model_QA_5_epoch_RU date: 2022-06-08 tags: [ru, open_source, distilbert, question_answering] task: Question Answering language: ru edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `model-QA-5-epoch-RU` is a Russian model originally trained by `AndrewChar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_model_QA_5_epoch_RU_ru_4.0.0_3.0_1654728395737.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_model_QA_5_epoch_RU_ru_4.0.0_3.0_1654728395737.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_model_QA_5_epoch_RU","ru") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Как меня зовут?", "Меня зовут Клара, и я живу в Беркли."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_model_QA_5_epoch_RU","ru") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Как меня зовут?", "Меня зовут Клара, и я живу в Беркли.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("ru.answer_question.distil_bert").predict("""Как меня зовут?|||"Меня зовут Клара, и я живу в Беркли.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_model_QA_5_epoch_RU| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ru| |Size:|505.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AndrewChar/model-QA-5-epoch-RU --- layout: model title: Chinese BertForQuestionAnswering model (from yechen) author: John Snow Labs name: bert_qa_question_answering_chinese date: 2022-06-02 tags: [zh, open_source, question_answering, bert] task: Question Answering language: zh edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `question-answering-chinese` is a Chinese model orginally trained by `yechen`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_chinese_zh_4.0.0_3.0_1654189170939.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_chinese_zh_4.0.0_3.0_1654189170939.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_question_answering_chinese","zh") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_question_answering_chinese","zh") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("zh.answer_question.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_question_answering_chinese| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|zh| |Size:|1.2 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/yechen/question-answering-chinese --- layout: model title: Persian BertForQuestionAnswering Cased model (from newsha) author: John Snow Labs name: bert_qa_pquad_2 date: 2022-07-07 tags: [fa, open_source, bert, question_answering] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `PQuAD_2` is a Persian model originally trained by `newsha`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_pquad_2_fa_4.0.0_3.0_1657182314348.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_pquad_2_fa_4.0.0_3.0_1657182314348.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_pquad_2","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_pquad_2","fa") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_pquad_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|fa| |Size:|607.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/newsha/PQuAD_2 --- layout: model title: Universal Sentence Encoder author: John Snow Labs name: tfhub_use date: 2020-04-17 task: Embeddings language: en nav_key: models edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: UniversalSentenceEncoder article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The Universal Sentence Encoder encodes text into high-dimensional vectors that can be used for text classification, semantic similarity, clustering and other natural language tasks. The model is trained and optimized for greater-than-word length text, such as sentences, phrases or short paragraphs. It is trained on a variety of data sources and a variety of tasks with the aim of dynamically accommodating a wide variety of natural language understanding tasks. The input is variable length English text and the output is a 512 dimensional vector. We apply this model to the STS benchmark for semantic similarity, and the results can be seen in the example notebook made available. The universal-sentence-encoder model is trained with a deep averaging network (DAN) encoder. The details are described in the paper "[Universal Sentence Encoder](https://arxiv.org/abs/1803.11175)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/tfhub_use_en_2.4.0_2.4_1587136330099.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/tfhub_use_en_2.4.0_2.4_1587136330099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = UniversalSentenceEncoder.pretrained("tfhub_use", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP', 'Many thanks']], ["text"])) ``` ```scala ... val embeddings = UniversalSentenceEncoder.pretrained("tfhub_use", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I love NLP", "Many thanks").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed_sentence.tfhub_use').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_tfhub_use_embeddings 0 I love NLP [0.06498772650957108, 0.01892215944826603, -0.... 1 Many thanks [0.0255892276763916, -0.042829226702451706, -0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|tfhub_use| |Type:|embeddings| |Compatibility:|Spark NLP 2.4.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://tfhub.dev/google/universal-sentence-encoder/2](https://tfhub.dev/google/universal-sentence-encoder/2) --- layout: model title: Pipeline to Extract Entities Related to TNM Staging author: John Snow Labs name: ner_oncology_tnm_pipeline date: 2023-03-09 tags: [licensed, en, clinical, oncology, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_tnm](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_tnm_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_tnm_pipeline_en_4.3.0_3.2_1678352273944.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_tnm_pipeline_en_4.3.0_3.2_1678352273944.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_tnm_pipeline", "en", "clinical/models") text = '''The final diagnosis was metastatic breast carcinoma, and it was classified as T2N1M1 stage IV. The histological grade of this 4 cm tumor was grade 2.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_tnm_pipeline", "en", "clinical/models") val text = "The final diagnosis was metastatic breast carcinoma, and it was classified as T2N1M1 stage IV. The histological grade of this 4 cm tumor was grade 2." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-----------------|--------:|------:|:------------------|-------------:| | 0 | metastatic | 24 | 33 | Metastasis | 0.9999 | | 1 | breast carcinoma | 35 | 50 | Cancer_Dx | 0.9972 | | 2 | T2N1M1 stage IV | 78 | 92 | Staging | 0.905267 | | 3 | 4 cm | 126 | 129 | Tumor_Description | 0.85105 | | 4 | tumor | 131 | 135 | Tumor | 0.9926 | | 5 | grade 2 | 141 | 147 | Tumor_Description | 0.89705 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_tnm_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Legal NER for NDA (Termination Clause) author: John Snow Labs name: legner_nda_termination date: 2023-04-06 tags: [en, licensed, legal, ner, nda, termination] task: Named Entity Recognition language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a NER model, aimed to be run **only** after detecting the `TERMINATION ` clause with a proper classifier (use legmulticlf_mnda_sections_paragraph_other for that purpose). It will extract the following entities: `TERM_DATE `, and `REF_TERM_DATE`. ## Predicted Entities `TERM_DATE`, `REF_TERM_DATE` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_nda_termination_en_1.0.0_3.0_1680816697135.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_nda_termination_en_1.0.0_3.0_1680816697135.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.RoBertaEmbeddings.pretrained("roberta_embeddings_legal_roberta_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_nda_termination", "en", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter]) empty_data = spark.createDataFrame([[""]]).toDF("text") model = nlpPipeline.fit(empty_data) text = ["""Except as otherwise specified herein, the obligations of the parties set forth in this Agreement shall terminate and be of no further force and effect eighteen months from the date hereof."""] result = model.transform(spark.createDataFrame([text]).toDF("text")) ```

## Results ```bash +---------------+-------------+ |chunk |ner_label | +---------------+-------------+ |eighteen months|TERM_DATE | |date hereof |REF_TERM_DATE| +---------------+-------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_nda_termination| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|16.2 MB| ## References In-house annotations on the Non-disclosure Agreements ## Benchmarking ```bash label precision recall f1-score support B-TERM_DATE 1.00 0.92 0.96 12 I-TERM_DATE 0.97 1.00 0.98 28 B-REF_TERM_DATE 0.91 0.91 0.91 11 I-REF_TERM_DATE 1.00 0.90 0.95 10 micro-avg 0.97 0.95 0.96 61 macro-avg 0.97 0.93 0.95 61 weighted-avg 0.97 0.95 0.96 61 ``` --- layout: model title: Danish asr_xls_r_300m_danish_nst_cv9 TFWav2Vec2ForCTC from chcaa author: John Snow Labs name: pipeline_asr_xls_r_300m_danish_nst_cv9 date: 2022-09-25 tags: [wav2vec2, da, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: da edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_300m_danish_nst_cv9` is a Danish model originally trained by chcaa. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_xls_r_300m_danish_nst_cv9_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_300m_danish_nst_cv9_da_4.2.0_3.0_1664102626776.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_xls_r_300m_danish_nst_cv9_da_4.2.0_3.0_1664102626776.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_xls_r_300m_danish_nst_cv9', lang = 'da') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_xls_r_300m_danish_nst_cv9", lang = "da") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_xls_r_300m_danish_nst_cv9| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|da| |Size:|757.6 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English financial Word Embeddings (Roberta) author: John Snow Labs name: roberta_embeddings_financial date: 2022-05-04 tags: [roberta, embeddings, en, open_source, financial] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Financial Pretrained Roberta Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `abhilash1910/financial_roberta` is a English model orginally trained by `abhilash1910`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_financial_en_3.4.2_3.0_1651678861262.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_financial_en_3.4.2_3.0_1651678861262.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_financial| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|324.5 MB| |Case sensitive:|true| --- layout: model title: Relation Extraction Between Dates and Clinical Entities (ReDL) author: John Snow Labs name: redl_date_clinical_biobert date: 2021-06-01 tags: [licensed, en, clinical, relation_extraction] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Identify if tests were conducted on a particular date or any diagnosis was made on a specific date by checking relations between clinical entities and dates. `1` : Shows date and the clinical entity are related, `0` : Shows date and the clinical entity are not related. ## Predicted Entities `0`, `1` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_CLINICAL_DATE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_date_clinical_biobert_en_3.0.3_3.0_1622583984107.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_date_clinical_biobert_en_3.0.3_3.0_1622583984107.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |---|-----------|---------|---------------|-------------|------------------------------------------|---------|-------------|-------------|---------|------------| | 0 | 1 | Test | 24 | 25 | CT | Date | 31 | 37 | 1/12/95 | 1.0 | | 1 | 1 | Symptom | 45 | 84 | progressive memory and cognitive decline | Date | 92 | 98 | 8/11/94 | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_date_clinical_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on an internal dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.738 0.729 0.734 84 1 0.945 0.947 0.946 416 Avg. 0.841 0.838 0.840 - ``` --- layout: model title: Malay DistilBERT Embeddings (from w11wo) author: John Snow Labs name: distilbert_embeddings_malaysian_distilbert_small date: 2022-04-12 tags: [distilbert, embeddings, ms, open_source] task: Embeddings language: ms edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: DistilBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `malaysian-distilbert-small` is a Malay model orginally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_malaysian_distilbert_small_ms_3.4.2_3.0_1649784041633.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_malaysian_distilbert_small_ms_3.4.2_3.0_1649784041633.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_malaysian_distilbert_small| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ms| |Size:|248.4 MB| |Case sensitive:|true| ## References - https://huggingface.co/w11wo/malaysian-distilbert-small - https://arxiv.org/abs/1910.01108 - https://github.com/sgugger - https://github.com/piegu/fastai-projects/blob/master/finetuning-English-GPT2-any-language-Portuguese-HuggingFace-fastaiv2.ipynb - https://w11wo.github.io/ --- layout: model title: Fast Neural Machine Translation Model from English to Bantu Languages author: John Snow Labs name: opus_mt_en_bnt date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, en, bnt, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `en` - target languages: `bnt` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_en_bnt_xx_2.7.0_2.4_1609169302611.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_en_bnt_xx_2.7.0_2.4_1609169302611.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_en_bnt| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Twi asr_wav2vec2large_xlsr_akan TFWav2Vec2ForCTC from azunre author: John Snow Labs name: asr_wav2vec2large_xlsr_akan date: 2022-09-24 tags: [wav2vec2, tw, audio, open_source, asr] task: Automatic Speech Recognition language: tw edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2large_xlsr_akan` is a Twi model originally trained by azunre. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2large_xlsr_akan_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2large_xlsr_akan_tw_4.2.0_3.0_1664022240589.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2large_xlsr_akan_tw_4.2.0_3.0_1664022240589.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2large_xlsr_akan", "tw")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2large_xlsr_akan", "tw") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2large_xlsr_akan| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|tw| |Size:|1.2 GB| --- layout: model title: Armenian Lemmatizer author: John Snow Labs name: lemma date: 2020-07-29 23:34:00 +0800 task: Lemmatization language: hy edition: Spark NLP 2.5.5 spark_version: 2.4 tags: [lemmatizer, hy] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb#scrollTo=bbzEH9u7tdxR){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_hy_2.5.5_2.4_1596054393298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_hy_2.5.5_2.4_1596054393298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... lemmatizer = LemmatizerModel.pretrained("lemma", "hy") \ .setInputCols(["token"]) \ .setOutputCol("lemma") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, lemmatizer]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար:") ``` ```scala ... val lemmatizer = LemmatizerModel.pretrained("lemma", "hy") .setInputCols(Array("token")) .setOutputCol("lemma") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, lemmatizer)) val data = Seq("Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար:").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Հյուսիսային թագավոր լինելուց բացի, Johnոն Սնոուն անգլիացի բժիշկ է և անզգայացման և բժշկական հիգիենայի զարգացման առաջատար:"""] lemma_df = nlu.load('hy.lemma').predict(text, output_level='document') lemma_df.lemma.values[0] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=10, result='Հյուսիսային', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=12, end=18, result='թագավոր', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=20, end=27, result='լինել', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=29, end=32, result='բացի', metadata={'sentence': '0'}, embeddings=[]), Row(annotatorType='token', begin=33, end=33, result=',', metadata={'sentence': '0'}, embeddings=[]), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Type:|lemmatizer| |Compatibility:|Spark NLP 2.5.5+| |Edition:|Official| |Input labels:|[token]| |Output labels:|[lemma]| |Language:|hy| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://universaldependencies.org](https://universaldependencies.org) --- layout: model title: Danish BertForQuestionAnswering model (from jacobshein) author: John Snow Labs name: bert_qa_danish_bert_botxo_qa_squad date: 2022-06-02 tags: [da, open_source, question_answering, bert] task: Question Answering language: da edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `danish-bert-botxo-qa-squad` is a Danish model orginally trained by `jacobshein`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_danish_bert_botxo_qa_squad_da_4.0.0_3.0_1654187371533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_danish_bert_botxo_qa_squad_da_4.0.0_3.0_1654187371533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_danish_bert_botxo_qa_squad","da") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_danish_bert_botxo_qa_squad","da") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("da.answer_question.squad.bert").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_danish_bert_botxo_qa_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|da| |Size:|412.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jacobshein/danish-bert-botxo-qa-squad - https://jacobhein.com/#contact - https://github.com/botxo/nordic_bert - https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/multilingual/squads-tar/da --- layout: model title: SDOH Housing Insecurity For Classification author: John Snow Labs name: genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli date: 2023-04-27 tags: [en, licensed, clinical, sdoh, housing, biobert, generic_classifier, housing_insecurity] task: Text Classification language: en edition: Healthcare NLP 4.3.2 spark_version: 3.0 supported: true annotator: GenericClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Generic Classifier model is intended for detecting whether the patient has housing insecurity. If the clinical note includes patient housing problems, the model identifies it. If there is no housing issue or it is not mentioned in the text, it is regarded as “no housing insecurity”. The model is trained by using GenericClassifierApproach annotator. `Housing_Insecurity`: The patient has housing problems. `No_Housing_Insecurity`: The patient has no housing problems or it is not mentioned in the clinical notes. ## Predicted Entities `Housing_Insecurity`, `No_Housing_Insecurity` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli_en_4.3.2_3.0_1682607884617.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli_en_4.3.2_3.0_1682607884617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------------------+-----------------------+ | text| result| +----------------------------------------------------------------------------------------------------+-----------------------+ |Patient: Mary H.\n\nBackground: Mary is a 40-year-old woman who has been diagnosed with asthma an...|[No_Housing_Insecurity]| |Patient: Sarah L.\n\nBackground: Sarah is a 35-year-old woman who has been experiencing housing i...| [Housing_Insecurity]| +----------------------------------------------------------------------------------------------------+-----------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_housing_insecurity_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.3.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH Project ## Benchmarking ```bash label precision recall f1-score support Housing_Insecurity 0.83 0.81 0.82 64 No_Housing_Insecurity 0.86 0.87 0.86 83 accuracy - - 0.84 147 macro-avg 0.84 0.84 0.84 147 weighted-avg 0.84 0.84 0.84 147 ``` --- layout: model title: Detect Living Species (bert_embeddings_bert_base_fr_cased) author: John Snow Labs name: ner_living_species_bert date: 2022-06-23 tags: [fr, ner, clinical, licensed, bert] task: Named Entity Recognition language: fr edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract living species from clinical texts in French which is critical to scientific disciplines like medicine, biology, ecology/biodiversity, nutrition and agriculture. This model is trained using `bert_embeddings_bert_base_fr_cased` embeddings. It is trained on the [LivingNER](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) corpus that is composed of clinical case reports extracted from miscellaneous medical specialties including COVID, oncology, infectious diseases, tropical medicine, urology, pediatrics, and others. **NOTE :** 1. The text files were translated from Spanish with a neural machine translation system. 2. The annotations were translated with the same neural machine translation system. 3. The translated annotations were transferred to the translated text files using an annotation transfer technology. ## Predicted Entities `HUMAN`, `SPECIES` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_fr_3.5.3_3.0_1655973142130.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_fr_3.5.3_3.0_1655973142130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_bert_base_fr_cased", "fr")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "fr", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) data = spark.createDataFrame([["""Femme de 47 ans allergique à l'iode, fumeuse sociale, opérée pour des varices, deux césariennes et un abcès fessier. Vit avec son mari et ses trois enfants, travaille comme enseignante. Initialement, le patient a eu une bonne évolution, mais au 2ème jour postopératoire, il a commencé à montrer une instabilité hémodynamique. Les sérologies pour Coxiella burnetii, Bartonella henselae, Borrelia burgdorferi, Entamoeba histolytica, Toxoplasma gondii, herpès simplex virus 1 et 2, cytomégalovirus, virus d'Epstein Barr, virus de la varicelle et du zona et parvovirus B19 étaient négatives. Cependant, un test au rose Bengale positif pour Brucella, le test de Coombs et les agglutinations étaient également positifs avec un titre de 1/40."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx") .setInputCols(Array("document")) .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols(Array("sentence")) .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_bert_base_fr_cased", "fr") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_living_species_bert", "fr", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter)) val data = Seq("""Femme de 47 ans allergique à l'iode, fumeuse sociale, opérée pour des varices, deux césariennes et un abcès fessier. Vit avec son mari et ses trois enfants, travaille comme enseignante. Initialement, le patient a eu une bonne évolution, mais au 2ème jour postopératoire, il a commencé à montrer une instabilité hémodynamique. Les sérologies pour Coxiella burnetii, Bartonella henselae, Borrelia burgdorferi, Entamoeba histolytica, Toxoplasma gondii, herpès simplex virus 1 et 2, cytomégalovirus, virus d'Epstein Barr, virus de la varicelle et du zona et parvovirus B19 étaient négatives. Cependant, un test au rose Bengale positif pour Brucella, le test de Coombs et les agglutinations étaient également positifs avec un titre de 1/40.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("fr.med_ner.living_species.bert").predict("""Femme de 47 ans allergique à l'iode, fumeuse sociale, opérée pour des varices, deux césariennes et un abcès fessier. Vit avec son mari et ses trois enfants, travaille comme enseignante. Initialement, le patient a eu une bonne évolution, mais au 2ème jour postopératoire, il a commencé à montrer une instabilité hémodynamique. Les sérologies pour Coxiella burnetii, Bartonella henselae, Borrelia burgdorferi, Entamoeba histolytica, Toxoplasma gondii, herpès simplex virus 1 et 2, cytomégalovirus, virus d'Epstein Barr, virus de la varicelle et du zona et parvovirus B19 étaient négatives. Cependant, un test au rose Bengale positif pour Brucella, le test de Coombs et les agglutinations étaient également positifs avec un titre de 1/40.""") ```

## Results ```bash +--------------------------------+-------+ |ner_chunk |label | +--------------------------------+-------+ |Femme |HUMAN | |mari |HUMAN | |enfants |HUMAN | |patient |HUMAN | |Coxiella burnetii |SPECIES| |Bartonella henselae |SPECIES| |Borrelia burgdorferi |SPECIES| |Entamoeba histolytica |SPECIES| |Toxoplasma gondii |SPECIES| |cytomégalovirus |SPECIES| |virus d'Epstein Barr |SPECIES| |virus de la varicelle et du zona|SPECIES| |parvovirus B19 |SPECIES| |Brucella |SPECIES| +--------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|fr| |Size:|16.4 MB| ## References [https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/](https://temu.bsc.es/livingner/2022/05/03/multilingual-corpus/) ## Benchmarking ```bash label precision recall f1-score support B-HUMAN 0.81 0.95 0.87 2549 B-SPECIES 0.66 0.87 0.75 2824 I-HUMAN 0.98 0.43 0.60 114 I-SPECIES 0.73 0.77 0.75 1109 micro-avg 0.73 0.87 0.80 6596 macro-avg 0.80 0.75 0.74 6596 weighted-avg 0.74 0.87 0.80 6596 ``` --- layout: model title: English RobertaForQuestionAnswering Cased model (from choosistant) author: John Snow Labs name: roberta_qa_model_fine_tuned date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `qa-model-fine-tuned` is a English model originally trained by `choosistant`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_model_fine_tuned_en_4.3.0_3.0_1674211892619.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_model_fine_tuned_en_4.3.0_3.0_1674211892619.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_model_fine_tuned","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_model_fine_tuned","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_model_fine_tuned| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/choosistant/qa-model-fine-tuned --- layout: model title: English DistilBertForQuestionAnswering model (from exafluence) author: John Snow Labs name: distilbert_qa_BERT_ClinicalQA date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `BERT-ClinicalQA` is a English model originally trained by `exafluence`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_BERT_ClinicalQA_en_4.0.0_3.0_1654722853170.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_BERT_ClinicalQA_en_4.0.0_3.0_1654722853170.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_BERT_ClinicalQA","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_BERT_ClinicalQA","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.clinical.distil_bert").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_BERT_ClinicalQA| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/exafluence/BERT-ClinicalQA --- layout: model title: Fast Neural Machine Translation Model from Japanese to English author: John Snow Labs name: opus_mt_jap_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, jap, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `jap` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_jap_en_xx_2.7.0_2.4_1609164435326.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_jap_en_xx_2.7.0_2.4_1609164435326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_jap_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Translate Congo Swahili to English Pipeline author: John Snow Labs name: translate_swc_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, swc, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `swc` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_swc_en_xx_2.7.0_2.4_1609687888047.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_swc_en_xx_2.7.0_2.4_1609687888047.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_swc_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_swc_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.swc.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_swc_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: German ALBERT Embeddings (from abhilash1910) author: John Snow Labs name: albert_embeddings_albert_german_ner date: 2022-04-14 tags: [albert, embeddings, de, open_source] task: Embeddings language: de edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: AlBertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained ALBERT Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `albert-german-ner` is a German model orginally trained by `abhilash1910`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_german_ner_de_3.4.2_3.0_1649954270550.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/albert_embeddings_albert_german_ner_de_3.4.2_3.0_1649954270550.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|albert_embeddings_albert_german_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|44.8 MB| |Case sensitive:|false| ## References - https://huggingface.co/abhilash1910/albert-german-ner --- layout: model title: Pipeline to Mapping SNOMED Codes with Their Corresponding UMLS Codes author: John Snow Labs name: snomed_umls_mapping date: 2023-06-13 tags: [en, licensed, clinical, pipeline, chunk_mapping, snomed, umls] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of `snomed_umls_mapper` model. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapping_en_4.4.4_3.2_1686663535191.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/snomed_umls_mapping_en_4.4.4_3.2_1686663535191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash Results | | snomed_code | umls_code | |---:|:---------------------------------|:-------------------------------| | 0 | 733187009 | 449433008 | 51264003 | C4546029 | C3164619 | C0271267 | {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|snomed_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|5.1 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: Context Spell Checker for English author: John Snow Labs name: spellcheck_dl date: 2022-03-28 tags: [spellcheck, en, open_source] task: Spell Check language: en nav_key: models edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true annotator: ContextSpellCheckerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Spell Checker is a sequence-to-sequence model that detects and corrects spelling errors in your input text. It’s based on Levenshtein Automaton for generating candidate corrections and a Neural Language Model for ranking corrections. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/CONTEXTUAL_SPELL_CHECKER/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/CONTEXTUAL_SPELL_CHECKER.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/spellcheck_dl_en_3.4.1_3.0_1648457196011.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/spellcheck_dl_en_3.4.1_3.0_1648457196011.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") tokenizer = RecursiveTokenizer()\ .setInputCols(["document"])\ .setOutputCol("token")\ .setPrefixes(["\"", "“", "(", "[", "\n", "."]) \ .setSuffixes(["\"", "”", ".", ",", "?", ")", "]", "!", ";", ":", "'s", "’s"]) spellModel = ContextSpellCheckerModel\ .pretrained("spellcheck_dl", "en")\ .setInputCols("token")\ .setOutputCol("checked")\ pipeline = Pipeline(stages = [documentAssembler, tokenizer, spellModel]) empty_df = spark.createDataFrame([[""]]).toDF("text") lp = LightPipeline(pipeline.fit(empty_df)) text = ["During the summer we have the best ueather.", "I have a black ueather jacket, so nice."] lp.annotate(text) ``` ```scala val assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new RecursiveTokenizer() .setInputCols(Array("document")) .setOutputCol("token") .setPrefixes(Array("\"", "“", "(", "[", "\n", ".")) .setSuffixes(Array("\"", "”", ".", ",", "?", ")", "]", "!", ";", ":", "'s", "’s")) val spellChecker = ContextSpellCheckerModel. pretrained("spellcheck_dl", "en"). setInputCols("token"). setOutputCol("checked") val pipeline = new Pipeline().setStages(Array(assembler, tokenizer, spellChecker)) val empty_df = spark.createDataFrame([[""]]).toDF("text") val lp = new LightPipeline(pipeline.fit(empty_df)) val text = Array("During the summer we have the best ueather.", "I have a black ueather jacket, so nice.") lp.annotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("spell").predict("""During the summer we have the best ueather.""") ```

## Results ```bash [{'checked': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'weather', '.'], 'document': ['During the summer we have the best ueather.'], 'token': ['During', 'the', 'summer', 'we', 'have', 'the', 'best', 'ueather', '.']}, {'checked': ['I', 'have', 'a', 'black', 'leather', 'jacket', ',', 'so', 'nice', '.'], 'document': ['I have a black ueather jacket, so nice.'], 'token': ['I', 'have', 'a', 'black', 'ueather', 'jacket', ',', 'so', 'nice', '.']}] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|spellcheck_dl| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[corrected]| |Language:|en| |Size:|99.7 MB| ## References Combination of custom data sets. --- layout: model title: RE Pipeline between Body Parts and Direction Entities author: John Snow Labs name: re_bodypart_directions_pipeline date: 2022-03-31 tags: [licensed, clinical, relation_extraction, body_part, directions, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [re_bodypart_directions](https://nlp.johnsnowlabs.com/2021/01/18/re_bodypart_directions_en.html) model. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_BODYPART_ENT/){:.button.button-orange.button-orange-trans.arr.button-icon} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.arr.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_pipeline_en_3.4.1_3.0_1648732927504.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_bodypart_directions_pipeline_en_3.4.1_3.0_1648732927504.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | index | relations | entity1 | entity1_begin | entity1_end | chunk1 | entity2 | entity2_end | entity2_end | chunk2 | confidence | |-------|-----------|-----------------------------|---------------|-------------|------------|-----------------------------|-------------|-------------|---------------|------------| | 0 | 1 | Direction | 35 | 39 | upper | Internal_organ_or_component | 41 | 50 | brain stem | 0.9999989 | | 1 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 59 | 68 | cerebellum | 0.99992585 | | 2 | 0 | Direction | 35 | 39 | upper | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.9999999 | | 3 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 54 | 57 | left | 0.999811 | | 4 | 0 | Internal_organ_or_component | 41 | 50 | brain stem | Direction | 75 | 79 | right | 0.9998203 | | 5 | 1 | Direction | 54 | 57 | left | Internal_organ_or_component | 59 | 68 | cerebellum | 1.0 | | 6 | 0 | Direction | 54 | 57 | left | Internal_organ_or_component | 81 | 93 | basil ganglia | 0.97616416 | | 7 | 0 | Internal_organ_or_component | 59 | 68 | cerebellum | Direction | 75 | 79 | right | 0.953046 | | 8 | 1 | Direction | 75 | 79 | right | Internal_organ_or_component | 81 | 93 | basil ganglia | 1.0 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_bodypart_directions_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - PerceptronModel - WordEmbeddingsModel - MedicalNerModel - NerConverter - DependencyParserModel - RelationExtractionModel --- layout: model title: Detect Adverse Drug Events (biobert) author: John Snow Labs name: ner_ade_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect adverse drug events in tweets, reviews, and medical text using pretrained NER model. ## Predicted Entities `DRUG`, `ADE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_ade_biobert_en_3.0.0_3.0_1617260850526.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_ade_biobert_en_3.0.0_3.0_1617260850526.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_ade_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val ner = MedicalNerModel.pretrained("ner_ade_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.ade_biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_ade_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash label precision recall f1-score support B-ADE 0.48 0.82 0.60 3582 B-DRUG 0.87 0.65 0.75 11763 I-ADE 0.48 0.76 0.59 4309 I-DRUG 0.95 0.28 0.43 7654 O 0.97 0.98 0.97 303457 accuracy 0.95 330765 macro-avg 0.75 0.70 0.67 330765 weighted-avg 0.95 0.95 0.94 330765 ``` --- layout: model title: English BertForQuestionAnswering model (from SupriyaArun) author: John Snow Labs name: bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-uncased-finetuned-squad` is a English model orginally trained by `SupriyaArun`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181043121.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad_en_4.0.0_3.0_1654181043121.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.base_uncased.by_SupriyaArun").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_SupriyaArun_bert_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/SupriyaArun/bert-base-uncased-finetuned-squad --- layout: model title: Legal Fees Clause Binary Classifier author: John Snow Labs name: legclf_fees_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `fees` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `fees` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_fees_clause_en_1.0.0_3.2_1660122434131.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_fees_clause_en_1.0.0_3.2_1660122434131.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_fees_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[fees]| |[other]| |[other]| |[fees]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_fees_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support fees 0.95 0.94 0.94 81 other 0.98 0.99 0.98 284 accuracy - - 0.98 365 macro-avg 0.97 0.96 0.96 365 weighted-avg 0.98 0.98 0.98 365 ``` --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_2_H_512_A_8_squad2 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-2_H-512_A-8_squad2` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_squad2_en_4.0.0_3.0_1654185232766.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_2_H_512_A_8_squad2_en_4.0.0_3.0_1654185232766.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_squad2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_2_H_512_A_8_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.bert.uncased_2l_512d_a8a_512d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_2_H_512_A_8_squad2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|83.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-2_H-512_A-8_squad2 --- layout: model title: Persian RoBERTa Embeddings (from HooshvareLab) author: John Snow Labs name: roberta_embeddings_roberta_fa_zwnj_base date: 2022-04-14 tags: [roberta, embeddings, fa, open_source] task: Embeddings language: fa edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `roberta-fa-zwnj-base` is a Persian model orginally trained by `HooshvareLab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_fa_zwnj_base_fa_3.4.2_3.0_1649948242326.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_roberta_fa_zwnj_base_fa_3.4.2_3.0_1649948242326.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_roberta_fa_zwnj_base| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|fa| |Size:|444.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/HooshvareLab/roberta-fa-zwnj-base - https://github.com/hooshvare/roberta/issues --- layout: model title: English ElectraForQuestionAnswering model (from mrm8488) author: John Snow Labs name: electra_qa_base_finetuned_squadv1 date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-finetuned-squadv1` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_finetuned_squadv1_en_4.0.0_3.0_1655920647215.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_finetuned_squadv1_en_4.0.0_3.0_1655920647215.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_finetuned_squadv1","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.electra.base.by_mrm8488").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/electra-base-finetuned-squadv1 - https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/ --- layout: model title: English image_classifier_vit_base_patch16_224_cifar10 ViTForImageClassification from karthiksv author: John Snow Labs name: image_classifier_vit_base_patch16_224_cifar10 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_patch16_224_cifar10` is a English model originally trained by karthiksv. ## Predicted Entities `deer`, `bird`, `dog`, `horse`, `automobile`, `truck`, `frog`, `ship`, `airplane`, `cat` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_cifar10_en_4.1.0_3.0_1660170286683.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_patch16_224_cifar10_en_4.1.0_3.0_1660170286683.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_patch16_224_cifar10", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_patch16_224_cifar10", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_patch16_224_cifar10| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English image_classifier_vit_base_beans_demo_v3 ViTForImageClassification from nateraw author: John Snow Labs name: image_classifier_vit_base_beans_demo_v3 date: 2022-08-10 tags: [vit, en, images, open_source] task: Image Classification language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: ViTForImageClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained VIT model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`image_classifier_vit_base_beans_demo_v3` is a English model originally trained by nateraw. ## Predicted Entities `angular_leaf_spot`, `bean_rust`, `healthy` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_demo_v3_en_4.1.0_3.0_1660167976186.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/image_classifier_vit_base_beans_demo_v3_en_4.1.0_3.0_1660167976186.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python image_assembler = ImageAssembler() \ .setInputCol("image") \ .setOutputCol("image_assembler") imageClassifier = ViTForImageClassification \ .pretrained("image_classifier_vit_base_beans_demo_v3", "en")\ .setInputCols("image_assembler") \ .setOutputCol("class") pipeline = Pipeline(stages=[ image_assembler, imageClassifier, ]) pipelineModel = pipeline.fit(imageDF) pipelineDF = pipelineModel.transform(imageDF) ``` ```scala val imageAssembler = new ImageAssembler() .setInputCol("image") .setOutputCol("image_assembler") val imageClassifier = ViTForImageClassification .pretrained("image_classifier_vit_base_beans_demo_v3", "en") .setInputCols("image_assembler") .setOutputCol("class") val pipeline = new Pipeline().setStages(Array(imageAssembler, imageClassifier)) val pipelineModel = pipeline.fit(imageDF) val pipelineDF = pipelineModel.transform(imageDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|image_classifier_vit_base_beans_demo_v3| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[image_assembler]| |Output Labels:|[class]| |Language:|en| |Size:|321.9 MB| --- layout: model title: English XlmRoBertaForQuestionAnswering (from teacookies) author: John Snow Labs name: xlm_roberta_qa_autonlp_roberta_base_squad2_24465519 date: 2022-06-23 tags: [en, open_source, question_answering, xlmroberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autonlp-roberta-base-squad2-24465519` is a English model originally trained by `teacookies`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465519_en_4.0.0_3.0_1655986545762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlm_roberta_qa_autonlp_roberta_base_squad2_24465519_en_4.0.0_3.0_1655986545762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = XlmRoBertaForQuestionAnswering.pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465519","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifier = XlmRoBertaForQuestionAnswering .pretrained("xlm_roberta_qa_autonlp_roberta_base_squad2_24465519","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.xlm_roberta.base_24465519.by_teacookies").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlm_roberta_qa_autonlp_roberta_base_squad2_24465519| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|887.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/teacookies/autonlp-roberta-base-squad2-24465519 --- layout: model title: English T5ForConditionalGeneration Small Cased model (from google) author: John Snow Labs name: t5_efficient_small_nl4 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-small-nl4` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl4_en_4.3.0_3.0_1675122566099.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_small_nl4_en_4.3.0_3.0_1675122566099.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_small_nl4","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_small_nl4","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_small_nl4| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|119.7 MB| ## References - https://huggingface.co/google/t5-efficient-small-nl4 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Mapping RxNorm Codes with Corresponding UMLS Codes author: John Snow Labs name: rxnorm_umls_mapper date: 2022-06-24 tags: [rxnorm, umls, chunk_mapper, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.3 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps RxNorm codes with corresponding UMLS Codes. ## Predicted Entities {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapper_en_3.5.3_3.0_1656088714126.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_umls_mapper_en_3.5.3_3.0_1656088714126.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("ner_chunk") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli", "en","clinical/models")\ .setInputCols(["ner_chunk"])\ .setOutputCol("sbert_embeddings") rxnorm_resolver = SentenceEntityResolverModel\ .pretrained("sbiobertresolve_rxnorm_augmented", "en", "clinical/models")\ .setInputCols(["ner_chunk", "sbert_embeddings"])\ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper = ChunkMapperModel.pretrained("rxnorm_umls_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("mappings")\ .setRels(["umls_code"]) pipeline = Pipeline( stages = [ documentAssembler, sbert_embedder, rxnorm_resolver, chunkerMapper ]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp= LightPipeline(model) result = lp.fullAnnotate(["amlodipine 5 MG", "hydrochlorothiazide 25 MG"]) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sbert_embeddings") val rxnorm_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_rxnorm_augmented", "en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper = ChunkMapperModel.pretrained("rxnorm_umls_mapper", "en", "clinical/models") .setInputCols(Array("rxnorm_code")) .setOutputCol("mappings") .setRels(Array("umls_code")) val pipeline = new Pipeline(stages = Array(documentAssembler, sbert_embedder, rxnorm_resolver, chunkerMapper) val model = pipeline.fit(Seq("").toDF("text")) val lp= LightPipeline(model) val result = lp.fullAnnotate(Seq("amlodipine 5 MG", "hydrochlorothiazide 25 MG")) ``` {:.nlu-block} ```python import nlu nlu.load("en.rxnorm_to_umls").predict("""hydrochlorothiazide 25 MG""") ```

## Results ```bash | | chunk | rxnorm_code | umls_mappings | |---:|:--------------------------|--------------:|:----------------| | 0 | amlodipine 5 MG | 329528 | C1124796 | | 1 | hydrochlorothiazide 25 MG | 310798 | C0977518 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_umls_mapper| |Compatibility:|Healthcare NLP 3.5.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[rxnorm_code]| |Output Labels:|[mappings]| |Language:|en| |Size:|1.9 MB| --- layout: model title: Detect bacterial species (embeddings_clinical_large) author: John Snow Labs name: ner_bacterial_species_emb_clinical_large date: 2023-05-23 tags: [ner, clinical, bacterial_species, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.2 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect different types of species of bacteria in text using pretrained NER model. ## Predicted Entities `SPECIES` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_BACTERIAL_SPECIES/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_bacterial_species_emb_clinical_large_en_4.4.2_3.0_1684854147723.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_bacterial_species_emb_clinical_large_en_4.4.2_3.0_1684854147723.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | begin | end | entities | |---:|:----------------------------|--------:|------:|:-----------| | 0 | Veillonella parvula | 16 | 34 | SPECIES | | 1 | Prevotella melaninogenica | 40 | 64 | SPECIES | | 2 | Streptococcus mitis | 148 | 166 | SPECIES | | 3 | S. oralis | 172 | 180 | SPECIES | | 4 | V. parvula | 464 | 473 | SPECIES | | 5 | P. melaninogenica | 476 | 492 | SPECIES | | 6 | Eikenella corrodens | 495 | 513 | SPECIES | | 7 | Neisseria mucosa | 516 | 531 | SPECIES | | 8 | Actinomyces odontolyticus | 534 | 558 | SPECIES | | 9 | Fusobacterium periodonticum | 561 | 587 | SPECIES | | 10 | F. nucleatum ss vincentii | 590 | 614 | SPECIES | | 11 | Porphyromonas gingivalis | 620 | 643 | SPECIES | | 12 | S. mitis | 703 | 710 | SPECIES | | 13 | S. oralis | 713 | 721 | SPECIES | | 14 | S. noxia | 727 | 734 | SPECIES | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_bacterial_species_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.2+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.8 MB| ## Benchmarking ```bash label precision recall f1-score support SPECIES 0.80 0.82 0.81 1810 micro-avg 0.80 0.82 0.81 1810 macro-avg 0.80 0.82 0.81 1810 weighted-avg 0.80 0.82 0.81 1810 ``` --- layout: model title: English T5ForConditionalGeneration Base Cased model (from google) author: John Snow Labs name: t5_efficient_base_dm512 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-base-dm512` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm512_en_4.3.0_3.0_1675110784924.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_base_dm512_en_4.3.0_3.0_1675110784924.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_base_dm512","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_base_dm512","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_base_dm512| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|316.6 MB| ## References - https://huggingface.co/google/t5-efficient-base-dm512 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: T5 for Passive to Active Style Transfer author: John Snow Labs name: t5_passive_to_active_styletransfer date: 2022-01-12 tags: [t5, open_source, en] task: Text Generation language: en nav_key: models edition: Spark NLP 3.4.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a text-to-text model based on T5 fine-tuned to generate passively written text from a actively written text input, for the task "transfer Passive to Active:". It is based on Prithiviraj Damodaran's Styleformer. ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/T5_LINGUISTIC/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5_LINGUISTIC.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_passive_to_active_styletransfer_en_3.4.0_3.0_1641987698487.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_passive_to_active_styletransfer_en_3.4.0_3.0_1641987698487.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import sparknlp from sparknlp.base import * from sparknlp.annotator import * spark = sparknlp.start() documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") t5 = T5Transformer.pretrained("t5_passive_to_active_styletransfer") \ .setTask("transfer Passive to Active:") \ .setInputCols(["documents"]) \ .setMaxOutputLength(200) \ .setOutputCol("transfers") pipeline = Pipeline().setStages([documentAssembler, t5]) data = spark.createDataFrame([["A letter was sent to you."]]).toDF("text") result = pipeline.fit(data).transform(data) result.select("transfers.result").show(truncate=False) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.seq2seq.T5Transformer import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer.pretrained("t5_passive_to_active_styletransfer") .setTask("transfer Passive to Active:") .setMaxOutputLength(200) .setInputCols("documents") .setOutputCol("transfer") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("A letter was sent to you.").toDF("text") val result = pipeline.fit(data).transform(data) result.select("transfer.result").show(false) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5.passive_to_active_styletransfer").predict("""transfer Passive to Active:""") ```

## Results ```bash +-------------------+ |result | +-------------------+ |[you sent a letter]| +-------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_passive_to_active_styletransfer| |Compatibility:|Spark NLP 3.4.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[transfers]| |Language:|en| |Size:|265.0 MB| ## Data Source The original model is from the transformers library: https://huggingface.co/prithivida/passive_to_active_styletransfer --- layout: model title: Financial Question Answering (Bert) author: John Snow Labs name: finqa_bert date: 2023-01-03 tags: [en, licensed] task: Question Answering language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Financial Bert-based Question Answering model, trained on squad-v2, finetuned on proprietary Financial questions and answers. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finqa_bert_en_1.0.0_3.0_1672759463237.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finqa_bert_en_1.0.0_3.0_1672759463237.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash `Nasdaq Global Select Market under the symbol CDNS` ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finqa_bert| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|407.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References Trained on squad-v2, finetuned on proprietary Financial questions and answers. --- layout: model title: Legal Disclosure Clause Binary Classifier author: John Snow Labs name: legclf_disclosure_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `disclosure` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `disclosure` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_disclosure_clause_en_1.0.0_3.2_1660122351983.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_disclosure_clause_en_1.0.0_3.2_1660122351983.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_disclosure_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[disclosure]| |[other]| |[other]| |[disclosure]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_disclosure_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.9 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support disclosure 0.93 0.82 0.87 33 other 0.93 0.98 0.95 86 accuracy - - 0.93 119 macro-avg 0.93 0.90 0.91 119 weighted-avg 0.93 0.93 0.93 119 ``` --- layout: model title: Adverse Drug Events Classifier (LogReg) author: John Snow Labs name: classifier_logreg_ade date: 2023-05-16 tags: [text_classification, ade, logreg, en, clinical, licensed] task: Text Classification language: en edition: Healthcare NLP 4.4.1 spark_version: 3.0 supported: true annotator: DocumentLogRegClassifierModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is trained with the Logistic Regression algorithm and classifies text/sentence into two categories: True : The sentence is talking about a possible ADE False : The sentence doesn’t have any information about an ADE. The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). ## Predicted Entities `True`, `False` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/classifier_logreg_ade_en_4.4.1_3.0_1684248428027.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/classifier_logreg_ade_en_4.4.1_3.0_1684248428027.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +----------------------------------------------------------------------------------------+-------+ |text |result | +----------------------------------------------------------------------------------------+-------+ |Detection of activated eosinophils in nasal polyps of an aspirin-induced asthma patient.|[True] | |None of the patients required treatment for the overdose. |[False]| +----------------------------------------------------------------------------------------+-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifier_logreg_ade| |Compatibility:|Healthcare NLP 4.4.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[prediction]| |Language:|en| |Size:|596.0 KB| ## References The corpus used for model training is ADE-Corpus-V2 Dataset: Adverse Drug Reaction Data. This is a dataset for classification of a sentence if it is ADE-related (True) or not (False). Reference: Gurulingappa et al., Benchmark Corpus to Support Information Extraction for Adverse Drug Effects, JBI, 2012. http://www.sciencedirect.com/science/article/pii/S1532046412000615 ## Benchmarking ```bash label precision recall f1-score support False 0.91 0.92 0.92 3362 True 0.79 0.79 0.79 1361 accuracy - - 0.88 4723 macro_avg 0.85 0.85 0.85 4723 weighted_avg 0.88 0.88 0.88 4723 ``` --- layout: model title: Explain Document pipeline for Hebrew (explain_document_lg) author: John Snow Labs name: explain_document_lg date: 2021-04-30 tags: [hebrew, ner, he, open_source, explain_document_lg, pipeline] task: Named Entity Recognition language: he edition: Spark NLP 3.0.2 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The explain_document_lg is a pre-trained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/explain_document_lg_he_3.0.2_3.0_1619775273050.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/explain_document_lg_he_3.0.2_3.0_1619775273050.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline('explain_document_lg', lang = 'he') annotations = pipeline.fullAnnotate(""היי, מעבדות ג'ון סנו!"")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("explain_document_lg", lang = "he") val result = pipeline.fullAnnotate("היי, מעבדות ג'ון סנו!")(0) ``` {:.nlu-block} ```python import nlu nlu.load("he.explain_document").predict("""היי, מעבדות ג'ון סנו!""") ```

## Results ```bash +----------------------+------------------------+----------------------+---------------------------+--------------------+---------+ | text| document| sentence| token| ner|ner_chunk| +----------------------+------------------------+----------------------+---------------------------+--------------------+---------+ | היי ג'ון מעבדות שלג! |[ היי ג'ון מעבדות שלג! ]|[היי ג'ון מעבדות שלג!]|[היי, ג'ון, מעבדות, שלג, !]|[O, B-PERS, O, O, O]| [ג'ון]| +----------------------+------------------------+----------------------+---------------------------+--------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|explain_document_lg| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.2+| |License:|Open Source| |Edition:|Official| |Language:|he| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - WordEmbeddingsModel - NerDLModel - NerConverter --- layout: model title: Multilingual XLMRobertaForTokenClassification Base Cased model (from dkasti) author: John Snow Labs name: xlmroberta_ner_dkasti_base_finetuned_panx_all date: 2022-08-13 tags: [xx, open_source, xlm_roberta, ner] task: Named Entity Recognition language: xx edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-all` is a Multilingual model originally trained by `dkasti`. ## Predicted Entities `ORG`, `LOC`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_dkasti_base_finetuned_panx_all_xx_4.1.0_3.0_1660428009273.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_dkasti_base_finetuned_panx_all_xx_4.1.0_3.0_1660428009273.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_dkasti_base_finetuned_panx_all","xx") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_dkasti_base_finetuned_panx_all","xx") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_dkasti_base_finetuned_panx_all| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|xx| |Size:|861.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/dkasti/xlm-roberta-base-finetuned-panx-all --- layout: model title: Notice Clause Binary Classifier author: John Snow Labs name: legclf_notice_clause date: 2022-12-16 tags: [en, legal, notice, classification, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `notice` clause type (where information about to people, addresses, notice methods, etc are mentioned). To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it’s better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Spark NLP for Legal Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `notice`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_notice_clause_en_1.0.0_3.0_1671208792323.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_notice_clause_en_1.0.0_3.0_1671208792323.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[notice]| |[other]| |[other]| |[notice]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_notice_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support notice 1.00 0.98 0.99 64 other 0.99 1.00 0.99 92 accuracy - - 0.99 156 macro-avg 0.99 0.99 0.99 156 weighted-avg 0.99 0.99 0.99 156 ``` --- layout: model title: English T5ForConditionalGeneration Small Cased model (from HUPD) author: John Snow Labs name: t5_hupd_small date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `hupd-t5-small` is a English model originally trained by `HUPD`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_hupd_small_en_4.3.0_3.0_1675102875669.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_hupd_small_en_4.3.0_3.0_1675102875669.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_hupd_small","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_hupd_small","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_hupd_small| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|289.1 MB| ## References - https://huggingface.co/HUPD/hupd-t5-small - https://patentdataset.org/ - https://github.com/suzgunmirac/hupd --- layout: model title: English DistilBertForQuestionAnswering model (from armageddon) author: John Snow Labs name: distilbert_qa_base_uncased_squad2_covid_qa_deepset date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-squad2-covid-qa-deepset` is a English model originally trained by `armageddon`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_covid_qa_deepset_en_4.0.0_3.0_1654727294594.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_squad2_covid_qa_deepset_en_4.0.0_3.0_1654727294594.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2_covid_qa_deepset","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_squad2_covid_qa_deepset","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid.distil_bert.base_uncased").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_squad2_covid_qa_deepset| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/armageddon/distilbert-base-uncased-squad2-covid-qa-deepset --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_de_4.2.0_3.0_1664117928306.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458_de_4.2.0_3.0_1664117928306.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2r_xls_r_accent_germany_2_austria_8_s458| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Exceptions Clause Binary Classifier author: John Snow Labs name: legclf_exceptions_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `exceptions` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `exceptions` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_exceptions_clause_en_1.0.0_3.2_1660123503661.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_exceptions_clause_en_1.0.0_3.2_1660123503661.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_exceptions_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[exceptions]| |[other]| |[other]| |[exceptions]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_exceptions_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.1 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support exceptions 0.88 0.94 0.91 32 other 0.97 0.95 0.96 79 accuracy - - 0.95 111 macro-avg 0.93 0.94 0.94 111 weighted-avg 0.95 0.95 0.95 111 ``` --- layout: model title: Icelandic RobertaForQuestionAnswering Cased model (from nozagleh) author: John Snow Labs name: roberta_qa_icebert_is_finetune date: 2023-01-20 tags: [is, open_source, roberta, question_answering, tensorflow] task: Question Answering language: is edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `IceBERT-QA-Is-finetune` is a Icelandic model originally trained by `nozagleh`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_is_finetune_is_4.3.0_3.0_1674208136981.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_icebert_is_finetune_is_4.3.0_3.0_1674208136981.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_icebert_is_finetune","is")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_icebert_is_finetune","is") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_icebert_is_finetune| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|is| |Size:|451.0 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/nozagleh/IceBERT-QA-Is-finetune --- layout: model title: Fast and Accurate Language Identification - 99 Languages (CNN) author: John Snow Labs name: ld_tatoeba_cnn_99 date: 2020-12-05 task: Language Detection language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [language_detection, open_source, xx] supported: true annotator: LanguageDetectorDL article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Language detection and identification is the task of automatically detecting the language(s) present in a document based on the content of the document. ``LanguageDetectorDL`` is an annotator that detects the language of documents or sentences depending on the ``inputCols``. In addition, ``LanguageDetetorDL`` can accurately detect language from documents with mixed languages by coalescing sentences and select the best candidate. We have designed and developed Deep Learning models using CNNs in TensorFlow/Keras. The model is trained on Tatoeba dataset with high accuracy evaluated on the Europarl dataset. The output is a language code in Wiki Code style: [https://en.wikipedia.org/wiki/List_of_Wikipedias](https://en.wikipedia.org/wiki/List_of_Wikipedias). This model can detect the following languages: `Afrikaans`, `Arabic`, `Algerian Arabic`, `Assamese`, `Kotava`, `Azerbaijani`, `Belarusian`, `Bengali`, `Berber`, `Breton`, `Bulgarian`, `Catalan`, `Chavacano`, `Cebuano`, `Czech`, `Chuvash`, `Mandarin Chinese`, `Cornish`, `Danish`, `German`, `Central Dusun`, `Modern Greek (1453-)`, `English`, `Esperanto`, `Estonian`, `Basque`, `Finnish`, `French`, `Guadeloupean Creole French`, `Irish`, `Galician`, `Gronings`, `Guarani`, `Hebrew`, `Hindi`, `Croatian`, `Hungarian`, `Armenian`, `Ido`, `Interlingue`, `Ilocano`, `Interlingua`, `Indonesian`, `Icelandic`, `Italian`, `Lojban`, `Japanese`, `Kabyle`, `Georgian`, `Kazakh`, `Khasi`, `Khmer`, `Korean`, `Coastal Kadazan`, `Latin`, `Lingua Franca Nova`, `Lithuanian`, `Latvian`, `Literary Chinese`, `Marathi`, `Meadow Mari`, `Macedonian`, `Low German (Low Saxon)`, `Dutch`, `Norwegian Nynorsk`, `Norwegian Bokmål`, `Occitan`, `Ottoman Turkish`, `Kapampangan`, `Picard`, `Persian`, `Polish`, `Portuguese`, `Romanian`, `Kirundi`, `Russian`, `Slovak`, `Spanish`, `Albanian`, `Serbian`, `Swedish`, `Swabian`, `Tatar`, `Tagalog`, `Thai`, `Klingon`, `Toki Pona`, `Turkmen`, `Turkish`, `Uyghur`, `Ukrainian`, `Urdu`, `Vietnamese`, `Volapük`, `Waray`, `Shanghainese`, `Yiddish`, `Cantonese`, `Malay`. ## Predicted Entities `af`, `ar`, `arq`, `as`, `avk`, `az`, `be`, `bn`, `ber`, `br`, `bg`, `ca`, `cbk`, `ceb`, `cs`, `cv`, `cmn`, `kw`, `da`, `de`, `dtp`, `el`, `en`, `eo`, `et`, `eu`, `fi`, `fr`, `gcf`, `ga`, `gl`, `gos`, `gn`, `he`, `hi`, `hr`, `hu`, `hy`, `io`, `ie`, `ilo`, `ia`, `id`, `is`, `it`, `jbo`, `ja`, `kab`, `ka`, `kk`, `kha`, `km`, `ko`, `kzj`, `la`, `lfn`, `lt`, `lvs`, `lzh`, `mr`, `mhr`, `mk`, `nds`, `nl`, `nn`, `nb`, `oc`, `ota`, `pam`, `pcd`, `pes`, `pl`, `pt`, `ro`, `rn`, `ru`, `sk`, `es`, `sq`, `sr`, `sv`, `swg`, `tt`, `tl`, `th`, `tlh`, `toki`, `tk`, `tr`, `ug`, `uk`, `ur`, `vi`, `vo`, `war`, `wuu`, `yi`, `yue`, `zsm`. {:.btn-box} [Open in Colab](https://githubtocolab.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/ld_tatoeba_cnn_99_xx_2.7.0_2.4_1607183215533.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/ld_tatoeba_cnn_99_xx_2.7.0_2.4_1607183215533.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... language_detector = LanguageDetectorDL.pretrained("ld_tatoeba_cnn_99", "xx")\ .setInputCols(["sentence"])\ .setOutputCol("language") languagePipeline = Pipeline(stages=[documentAssembler, sentenceDetector, language_detector]) light_pipeline = LightPipeline(languagePipeline.fit(spark.createDataFrame([['']]).toDF("text"))) result = light_pipeline.fullAnnotate("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.") ``` ```scala ... val languageDetector = LanguageDetectorDL.pretrained("ld_tatoeba_cnn_99", "xx") .setInputCols("sentence") .setOutputCol("language") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, languageDetector)) val data = Seq("Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."] lang_df = nlu.load('xx.classify.wiki_99').predict(text, output_level='sentence') lang_df ```

## Results ```bash 'fr' ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ld_tatoeba_cnn_99| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[language]| |Language:|xx| ## Data Source Tatoeba ## Benchmarking ```bash Evaluated on Europarl dataset which the model has never seen: +--------+-----+-------+------------------+ |src_lang|count|correct| precision| +--------+-----+-------+------------------+ | fi| 1000| 1000| 1.0| | pt| 1000| 1000| 1.0| | it| 1000| 998| 0.998| | es| 1000| 997| 0.997| | en| 1000| 995| 0.995| | da| 1000| 994| 0.994| | sv| 1000| 992| 0.992| | pl| 914| 899|0.9835886214442013| | hu| 880| 863|0.9806818181818182| | lt| 1000| 975| 0.975| | bg| 1000| 951| 0.951| | et| 928| 783| 0.84375| +--------+-----+-------+------------------+ +-------+-------------------+ |summary| precision| +-------+-------------------+ | count| 12| | mean| 0.9758350366355016| | stddev|0.04391442353856736| | min| 0.84375| | max| 1.0| +-------+-------------------+ ``` --- layout: model title: Detect Clinical Entities (ner_jsl_greedy_biobert) author: John Snow Labs name: ner_jsl_greedy_biobert date: 2021-08-13 tags: [ner, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.2.0 spark_version: 2.4 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained named entity recognition deep learning model for clinical terminology. The SparkNLP deep learning model (MedicalNerModel) is inspired by a former state of the art model for NER: Chiu & Nicols, Named Entity Recognition with Bidirectional LSTM-CNN. This model is the official version of jsl_ner_wip_greedy_biobert model. Definitions of Predicted Entities: - `Injury_or_Poisoning`: Physical harm or injury caused to the body, including those caused by accidents, falls, or poisoning of a patient or someone else. - `Direction`: All the information relating to the laterality of the internal and external organs. - `Test`: Mentions of laboratory, pathology, and radiological tests. - `Admission_Discharge`: Terms that indicate the admission and/or the discharge of a patient. - `Death_Entity`: Mentions that indicate the death of a patient. - `Relationship_Status`: State of patients romantic or social relationships (e.g. single, married, divorced). - `Duration`: The duration of a medical treatment or medication use. - `Respiration`: Number of breaths per minute. - `Hyperlipidemia`: Terms that indicate hyperlipidemia with relevant subtypes and synonims. - `Birth_Entity`: Mentions that indicate giving birth. - `Age`: All mention of ages, past or present, related to the patient or with anybody else. - `Labour_Delivery`: Extractions include stages of labor and delivery. - `Family_History_Header`: identifies section headers that correspond to Family History of the patient. - `BMI`: Numeric values and other text information related to Body Mass Index. - `Temperature`: All mentions that refer to body temperature. - `Alcohol`: Terms that indicate alcohol use, abuse or drinking issues of a patient or someone else. - `Kidney_Disease`: Terms that refer to any kidney diseases (includes mentions of modifiers such as "Acute" or "Chronic"). - `Oncological`: All the cancer, tumor or metastasis related extractions mentioned in the document, of the patient or someone else. - `Medical_History_Header`: Identifies section headers that correspond to Past Medical History of a patient. - `Cerebrovascular_Disease`: All terms that refer to cerebrovascular diseases and events. - `Oxygen_Therapy`: Breathing support triggered by patient or entirely or partially by machine (e.g. ventilator, BPAP, CPAP). - `O2_Saturation`: Systemic arterial, venous or peripheral oxygen saturation measurements. - `Psychological_Condition`: All the Mental health diagnosis, disorders, conditions or syndromes of a patient or someone else. - `Heart_Disease`: All mentions of acquired, congenital or degenerative heart diseases. - `Employment`: All mentions of patient or provider occupational titles and employment status . - `Obesity`: Terms related to a patient being obese (overweight and BMI are extracted as different labels). - `Disease_Syndrome_Disorder`: All the diseases mentioned in the document, of the patient or someone else (excluding diseases that are extracted with their specific labels, such as "Heart_Disease" etc.). - `Pregnancy`: All terms related to Pregnancy (excluding terms that are extracted with their specific labels, such as "Labour_Delivery" etc.). - `ImagingFindings`: All mentions of radiographic and imagistic findings. - `Procedure`: All mentions of invasive medical or surgical procedures or treatments. - `Medical_Device`: All mentions related to medical devices and supplies. - `Race_Ethnicity`: All terms that refer to racial and national origin of sociocultural groups. - `Section_Header`: All the section headers present in the text (Medical History, Family History, Social History, Physical Examination and Vital signs Headers are extracted separately with their specific labels). - `Symptom`: All the symptoms mentioned in the document, of a patient or someone else. - `Treatment`: Includes therapeutic and minimally invasive treatment and procedures (invasive treatments or procedures are extracted as "Procedure"). - `Substance`: All mentions of substance use related to the patient or someone else (recreational drugs, illicit drugs). - `Route`: Drug and medication administration routes available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_Ingredient`: Active ingredient/s found in drug products. - `Blood_Pressure`: Systemic blood pressure, mean arterial pressure, systolic and/or diastolic are extracted. - `Diet`: All mentions and information regarding patients dietary habits. - `External_body_part_or_region`: All mentions related to external body parts or organs that can be examined by naked eye. - `LDL`: All mentions related to the lab test and results for LDL (Low Density Lipoprotein). - `VS_Finding`: Qualitative data (e.g. Fever, Cyanosis, Tachycardia) and any other symptoms that refers to vital signs. - `Allergen`: Allergen related extractions mentioned in the document. - `EKG_Findings`: All mentions of EKG readings. - `Imaging_Technique`: All mentions of special radiographic views or special imaging techniques used in radiology. - `Triglycerides`: All mentions terms related to specific lab test for Triglycerides. - `RelativeTime`: Time references that are relative to different times or events (e.g. words such as "approximately", "in the morning"). - `Gender`: Gender-specific nouns and pronouns. - `Pulse`: Peripheral heart rate, without advanced information like measurement location. - `Social_History_Header`: Identifies section headers that correspond to Social History of a patient. - `Substance_Quantity`: All mentions of substance quantity (quantitative information related to illicit/recreational drugs). - `Diabetes`: All terms related to diabetes mellitus. - `Modifier`: Terms that modify the symptoms, diseases or risk factors. If a modifier is included in ICD-10 name of a specific disease, the respective modifier is not extracted separately. - `Internal_organ_or_component`: All mentions related to internal body parts or organs that can not be examined by naked eye. - `Clinical_Dept`: Terms that indicate the medical and/or surgical departments. - `Form`: Drug and medication forms available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Drug_BrandName`: Commercial labeling name chosen by the labeler or the drug manufacturer for a drug containing a single or multiple drug active ingredients. - `Strength`: Potency of one unit of drug (or a combination of drugs) the measurement units available are described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Fetus_NewBorn`: All terms related to fetus, infant, new born (excluding terms that are extracted with their specific labels, such as "Labour_Delivery", "Pregnancy" etc.). - `RelativeDate`: Temporal references that are relative to the date of the text or to any other specific date (e.g. "approximately two years ago", "about two days ago"). - `Height`: All mentions related to a patients height. - `Test_Result`: Terms related to all the test results present in the document (clinical tests results are included). - `Sexually_Active_or_Sexual_Orientation`: All terms that are related to sexuality, sexual orientations and sexual activity. - `Frequency`: Frequency of administration for a dose prescribed. - `Time`: Specific time references (hour and/or minutes). - `Weight`: All mentions related to a patients weight. - `Vaccine`: Generic and brand name of vaccines or vaccination procedure. - `Vital_Signs_Header`: Identifies section headers that correspond to Vital Signs of a patient. - `Communicable_Disease`: Includes all mentions of communicable diseases. - `Dosage`: Quantity prescribed by the physician for an active ingredient; measurement units are available described by [FDA](http://wayback.archive-it.org/7993/20171115111313/https:/www.fda.gov/Drugs/DevelopmentApprovalProcess/FormsSubmissionRequirements/ElectronicSubmissions/DataStandardsManualmonographs/ucm071667.htm). - `Overweight`: Terms related to the patient being overweight (BMI and Obesity is extracted separately). - `Hypertension`: All terms related to Hypertension (quantitative data such as 150/100 is extracted as Blood_Pressure). - `HDL`: Terms related to the lab test for HDL (High Density Lipoprotein). - `Total_Cholesterol`: Terms related to the lab test and results for cholesterol. - `Smoking`: All mentions of smoking status of a patient. - `Date`: Mentions of an exact date, in any format, including day number, month and/or year. ## Predicted Entities `Test_Result`, `Relationship_Status`, `RelativeDate`, `Blood_Pressure`, `Triglycerides`, `Smoking`, `Pregnancy`, `Medical_History_Header`, `LDL`, `Hypertension`, `Hyperlipidemia`, `Frequency`, `BMI`, `Internal_organ_or_component`, `Allergen`, `Fetus_NewBorn`, `Substance_Quantity`, `Time`, `Temperature`, `Procedure`, `Strength`, `Treatment`, `HDL`, `Alcohol`, `Birth_Entity`, `Diet`, `Weight`, `Oxygen_Therapy`, `Injury_or_Poisoning`, `Section_Header`, `Obesity`, `EKG_Findings`, `Gender`, `Height`, `Social_History_Header`, `Diabetes`, `Route`, `Race_Ethnicity`, `Substance`, `Drug`, `External_body_part_or_region`, `RelativeTime`, `Admission_Discharge`, `Psychological_Condition`, `Total_Cholesterol`, `Labour_Delivery`, `Imaging_Technique`, `Date`, `Form`, `Overweight`, `Cerebrovascular_Disease`, `Vital_Signs_Header`, `Oncological`, `ImagingFindings`, `Communicable_Disease`, `Duration`, `Vaccine`, `Kidney_Disease`, `O2_Saturation`, `Heart_Disease`, `Employment`, `Sexually_Active_or_Sexual_Orientation`, `Test`, `Disease_Syndrome_Disorder`, `Respiration`, `Direction`, `Medical_Device`, `Clinical_Dept`, `Modifier`, `Symptom`, `Pulse`, `Age`, `Death_Entity`, `Dosage`, `Family_History_Header`, `VS_Finding` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_JSL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_JSL.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_en_3.2.0_2.4_1628850116918.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_greedy_biobert_en_3.2.0_2.4_1628850116918.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunk | entity | |---:|:-----------------------------------------------|:-----------------------------| | 0 | 21-day-old | Age | | 1 | Caucasian | Race_Ethnicity | | 2 | male | Gender | | 3 | for 2 days | Duration | | 4 | congestion | Symptom | | 5 | mom | Gender | | 6 | suctioning yellow discharge | Symptom | | 7 | nares | External_body_part_or_region | | 8 | she | Gender | | 9 | mild problems with his breathing while feeding | Symptom | | 10 | perioral cyanosis | Symptom | | 11 | retractions | Symptom | | 12 | One day ago | RelativeDate | | 13 | mom | Gender | | 14 | tactile temperature | Symptom | | 15 | Tylenol | Drug | | 16 | Baby | Age | | 17 | decreased p.o. intake | Symptom | | 18 | His | Gender | | 19 | breast-feeding | External_body_part_or_region | | 20 | q.2h | Frequency | | 21 | to 5 to 10 minutes | Duration | | 22 | his | Gender | | 23 | respiratory congestion | Symptom | | 24 | He | Gender | | 25 | tired | Symptom | | 26 | fussy | Symptom | | 27 | over the past 2 days | RelativeDate | | 28 | albuterol | Drug | | 29 | ER | Clinical_Dept | | 30 | His | Gender | | 31 | urine output has also decreased | Symptom | | 32 | he | Gender | | 33 | per 24 hours | Frequency | | 34 | he | Gender | | 35 | per 24 hours | Frequency | | 36 | Mom | Gender | | 37 | diarrhea | Symptom | | 38 | His | Gender | | 39 | bowel | Internal_organ_or_component | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_greedy_biobert| |Compatibility:|Healthcare NLP 3.2.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Data Source Trained on data gathered and manually annotated by John Snow Labs. https://www.johnsnowlabs.com/data/ ## Benchmarking ```bash label tp fp fn prec rec f1 B-Oxygen_Therapy 47 11 10 0.8103448 0.8245614 0.81739134 B-Cerebrovascular_Disease 43 20 21 0.6825397 0.671875 0.6771653 B-Triglycerides 5 0 0 1 1 1 I-Cerebrovascular_Disease 25 12 27 0.6756757 0.48076922 0.56179774 B-Medical_Device 2704 531 364 0.8358578 0.88135594 0.85800415 B-Labour_Delivery 43 16 29 0.7288136 0.5972222 0.6564886 I-Vaccine 5 0 5 1 0.5 0.6666667 I-Obesity 6 4 1 0.6 0.85714287 0.70588243 I-Smoking 3 1 2 0.75 0.6 0.6666667 B-RelativeTime 67 36 51 0.65048546 0.5677966 0.60633487 B-Imaging_Technique 33 12 19 0.73333335 0.63461536 0.68041235 B-Heart_Disease 285 55 68 0.8382353 0.8073654 0.82251084 B-Procedure 1876 303 384 0.8609454 0.8300885 0.84523547 I-RelativeTime 105 43 53 0.7094595 0.664557 0.6862745 B-Drug 1803 299 265 0.8577545 0.87185687 0.8647482 B-Obesity 29 9 5 0.7631579 0.85294116 0.8055555 I-RelativeDate 617 167 107 0.7869898 0.8522099 0.8183024 B-O2_Saturation 27 8 6 0.7714286 0.8181818 0.7941177 B-Direction 2856 390 326 0.8798521 0.89754874 0.88861233 I-Alcohol 4 4 4 0.5 0.5 0.5 I-Oxygen_Therapy 25 7 6 0.78125 0.8064516 0.79365087 B-Diet 23 14 32 0.6216216 0.4181818 0.5 B-Dosage 35 26 29 0.57377046 0.546875 0.55999994 B-Injury_or_Poisoning 308 52 83 0.85555553 0.7877238 0.82023966 B-Hypertension 80 9 2 0.8988764 0.9756098 0.9356726 I-Test_Result 124 73 156 0.6294416 0.44285715 0.5199161 B-Alcohol 54 11 12 0.83076924 0.8181818 0.8244275 B-Height 14 5 5 0.7368421 0.7368421 0.7368421 I-Substance 18 8 8 0.6923077 0.6923077 0.6923077 B-RelativeDate 372 109 93 0.7733888 0.8 0.78646934 B-Admission_Discharge 218 22 14 0.90833336 0.9396552 0.9237288 B-Date 345 24 26 0.93495935 0.9299191 0.9324324 B-Kidney_Disease 63 10 20 0.8630137 0.7590361 0.8076923 I-Strength 22 17 13 0.5641026 0.62857145 0.59459466 I-Injury_or_Poisoning 301 93 98 0.7639594 0.75438595 0.75914246 I-Time 28 11 17 0.71794873 0.62222224 0.6666667 B-Substance 48 11 10 0.8135593 0.82758623 0.8205129 B-Total_Cholesterol 6 3 0 0.6666667 1 0.8 I-Vital_Signs_Header 276 28 8 0.90789473 0.97183096 0.93877554 I-Internal_organ_or_component 2907 518 490 0.8487591 0.8557551 0.8522427 B-Hyperlipidemia 28 3 0 0.9032258 1 0.9491525 B-Overweight 3 0 3 1 0.5 0.6666667 I-Sexually_Active_or_Sexual_Orientation 2 0 3 1 0.4 0.5714286 B-Sexually_Active_or_Sexual_Orientation 2 0 2 1 0.5 0.6666667 I-Fetus_NewBorn 50 38 58 0.5681818 0.46296296 0.5102041 B-BMI 6 0 1 1 0.85714287 0.9230769 B-ImagingFindings 52 41 61 0.5591398 0.460177 0.5048544 B-Test_Result 714 135 212 0.8409894 0.7710583 0.8045071 B-Section_Header 2140 79 65 0.9643984 0.97052157 0.96745026 I-Treatment 85 21 29 0.8018868 0.74561405 0.7727273 B-Clinical_Dept 638 82 77 0.88611114 0.8923077 0.88919866 I-Kidney_Disease 114 7 18 0.94214875 0.8636364 0.90118575 I-Pulse 189 27 42 0.875 0.8181818 0.84563756 B-Test 1589 320 315 0.83237296 0.83455884 0.83346444 B-Weight 54 12 13 0.8181818 0.80597013 0.81203 I-Respiration 114 4 17 0.9661017 0.870229 0.91566265 I-EKG_Findings 68 34 52 0.6666667 0.56666666 0.6126126 I-Section_Header 3828 168 77 0.957958 0.9802817 0.9689913 B-Strength 27 13 23 0.675 0.54 0.6 I-Social_History_Header 137 4 4 0.9716312 0.9716312 0.9716312 B-Vital_Signs_Header 183 18 7 0.9104478 0.9631579 0.9360614 B-Death_Entity 28 9 6 0.7567568 0.8235294 0.7887324 B-Modifier 302 90 282 0.77040815 0.5171233 0.6188525 B-Blood_Pressure 93 14 21 0.86915886 0.81578946 0.84162897 I-O2_Saturation 49 19 23 0.7205882 0.6805556 0.7 B-Frequency 437 77 68 0.8501946 0.86534655 0.8577036 I-Triglycerides 5 0 0 1 1 1 I-Duration 513 254 47 0.66883963 0.9160714 0.77317256 I-Diabetes 50 4 6 0.9259259 0.89285713 0.90909094 B-Race_Ethnicity 78 3 2 0.962963 0.975 0.9689441 I-Gender 114 2 17 0.98275864 0.870229 0.9230769 I-Height 43 13 10 0.76785713 0.8113208 0.78899086 B-Communicable_Disease 10 5 9 0.6666667 0.5263158 0.5882354 I-Family_History_Header 134 1 0 0.9925926 1 0.9962825 B-LDL 2 2 2 0.5 0.5 0.5 I-Race_Ethnicity 6 0 0 1 1 1 B-Psychological_Condition 103 21 17 0.83064514 0.85833335 0.84426236 I-Age 116 14 50 0.8923077 0.6987952 0.78378385 B-EKG_Findings 33 18 32 0.64705884 0.50769234 0.56896555 B-Employment 168 29 44 0.8527919 0.7924528 0.8215159 I-Oncological 358 38 17 0.9040404 0.9546667 0.9286641 B-Time 27 7 18 0.7941176 0.6 0.68354434 B-Treatment 93 31 41 0.75 0.69402987 0.7209303 B-Temperature 69 5 8 0.9324324 0.8961039 0.9139073 I-Procedure 2437 379 501 0.86541194 0.8294758 0.84706295 B-Relationship_Status 30 3 1 0.90909094 0.9677419 0.9375 B-Pregnancy 56 17 30 0.7671233 0.6511628 0.7044025 I-Route 8 4 7 0.6666667 0.53333336 0.59259266 I-Medical_History_Header 151 4 15 0.9741936 0.9096386 0.94080997 I-Imaging_Technique 25 5 20 0.8333333 0.5555556 0.66666675 B-Smoking 74 6 4 0.925 0.94871795 0.93670887 I-Labour_Delivery 36 8 18 0.8181818 0.6666667 0.7346939 I-Death_Entity 3 0 2 1 0.6 0.75 B-Diabetes 77 9 5 0.89534885 0.9390244 0.9166666 B-Gender 4479 82 111 0.9820215 0.97581697 0.9789094 B-Vaccine 6 1 9 0.85714287 0.4 0.54545456 I-Heart_Disease 393 61 89 0.8656388 0.8153527 0.8397436 I-Dosage 31 27 22 0.5344828 0.5849057 0.5585586 B-Social_History_Header 78 2 3 0.975 0.962963 0.9689441 B-External_body_part_or_region 1640 402 311 0.8031342 0.8405946 0.8214376 I-Clinical_Dept 546 59 47 0.90247935 0.920742 0.91151917 I-Test 1195 320 402 0.7887789 0.748278 0.7679949 I-Frequency 340 97 120 0.77803206 0.73913044 0.75808245 B-Age 454 35 57 0.9284254 0.888454 0.908 B-Pulse 90 11 17 0.8910891 0.8411215 0.8653846 I-Symptom 4265 2050 1232 0.6753761 0.7758778 0.72214705 I-Pregnancy 39 28 42 0.58208954 0.4814815 0.527027 I-LDL 5 0 4 1 0.5555556 0.71428573 I-Diet 33 14 25 0.70212764 0.5689655 0.6285714 I-Blood_Pressure 198 54 27 0.78571427 0.88 0.83018863 I-ImagingFindings 136 99 85 0.57872343 0.61538464 0.5964913 I-Date 203 13 10 0.9398148 0.9530516 0.946387 B-Route 84 23 47 0.78504676 0.64122134 0.7058824 B-Duration 204 110 26 0.6496815 0.8869565 0.74999994 B-Medical_History_Header 56 1 7 0.98245615 0.8888889 0.93333334 B-Respiration 55 4 6 0.9322034 0.90163934 0.9166667 I-External_body_part_or_region 314 105 167 0.74940336 0.65280664 0.6977778 I-BMI 15 0 1 1 0.9375 0.9677419 B-Internal_organ_or_component 4349 886 761 0.8307545 0.8510763 0.8407926 I-Weight 150 22 23 0.872093 0.867052 0.8695652 B-Disease_Syndrome_Disorder 1698 375 358 0.81910276 0.82587546 0.8224752 B-Symptom 4358 1002 932 0.8130597 0.8238185 0.8184037 B-VS_Finding 138 36 37 0.79310346 0.7885714 0.79083097 I-Disease_Syndrome_Disorder 1723 372 451 0.82243437 0.7925483 0.8072148 I-Drug 3282 838 493 0.79660195 0.86940396 0.8314123 I-Medical_Device 1864 418 242 0.81682736 0.88509023 0.84958977 B-Oncological 278 22 22 0.9266667 0.9266667 0.9266667 I-Temperature 111 8 6 0.9327731 0.94871795 0.94067794 I-Employment 92 27 19 0.77310926 0.8288288 0.8 I-Psychological_Condition 32 7 19 0.82051283 0.627451 0.7111111 B-Family_History_Header 68 0 0 1 1 1 I-Direction 311 91 144 0.7736318 0.6835165 0.72578764 Macro-average 65035 12855 11898 0.761429 0.706300 0.7328297 Micro-average 65035 12855 11898 0.834959 0.845346 0.8401207 ``` --- layout: model title: English asr_wav2vec2_base_timit_demo_colab30 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab30 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab30` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab30_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab30_en_4.2.0_3.0_1664020634835.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab30_en_4.2.0_3.0_1664020634835.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab30", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab30", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab30| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: Part of Speech for Basque author: John Snow Labs name: pos_ud_bdt date: 2021-03-09 tags: [part_of_speech, open_source, basque, pos_ud_bdt, eu] task: Part of Speech Tagging language: eu edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true annotator: PerceptronModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description A [Part of Speech](https://en.wikipedia.org/wiki/Part_of_speech) classifier predicts a grammatical label for every token in the input text. Implemented with an `averaged perceptron architecture`. ## Predicted Entities - ADV - PUNCT - VERB - NOUN - NUM - CCONJ - ADJ - AUX - PROPN - DET - PRON - PART - SYM - INTJ - X - ADP {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/GRAMMAR_EN/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/GRAMMAR_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pos_ud_bdt_eu_3.0.0_3.0_1615292144964.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pos_ud_bdt_eu_3.0.0_3.0_1615292144964.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") pos = PerceptronModel.pretrained("pos_ud_bdt", "eu") \ .setInputCols(["document", "token"]) \ .setOutputCol("pos") pipeline = Pipeline(stages=[ document_assembler, sentence_detector, posTagger ]) example = spark.createDataFrame([['Kaixo John Snow Labs-etik! ']], ["text"]) result = pipeline.fit(example).transform(example) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val pos = PerceptronModel.pretrained("pos_ud_bdt", "eu") .setInputCols(Array("document", "token")) .setOutputCol("pos") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, pos)) val data = Seq("Kaixo John Snow Labs-etik! ").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = [""Kaixo John Snow Labs-etik! ""] token_df = nlu.load('eu.pos').predict(text) token_df ```

## Results ```bash token pos 0 Kaixo INTJ 1 John PROPN 2 Snow PROPN 3 Labs-etik PROPN 4 ! PUNCT ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pos_ud_bdt| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[pos]| |Language:|eu| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab2_by_sameearif88 TFWav2Vec2ForCTC from sameearif88 author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab2_by_sameearif88` is a English model originally trained by sameearif88. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88_en_4.2.0_3.0_1664040818102.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88_en_4.2.0_3.0_1664040818102.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab2_by_sameearif88| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Chinese T5ForConditionalGeneration Base Cased model (from thu-coai) author: John Snow Labs name: t5_longlm_base date: 2023-01-30 tags: [zh, open_source, t5] task: Text Generation language: zh edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `LongLM-base` is a Chinese model originally trained by `thu-coai`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_longlm_base_zh_4.3.0_3.0_1675098192540.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_longlm_base_zh_4.3.0_3.0_1675098192540.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_longlm_base","zh") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_longlm_base","zh") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_longlm_base| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|zh| |Size:|926.8 MB| ## References - https://huggingface.co/thu-coai/LongLM-base - https://jianguanthu.github.io/ - http://coai.cs.tsinghua.edu.cn/ --- layout: model title: English DistilBertForTokenClassification Cased model (from ismail-lucifer011) author: John Snow Labs name: distilbert_token_classifier_autotrain_name_vsv_all_901529445 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-name_vsv_all-901529445` is a English model originally trained by `ismail-lucifer011`. ## Predicted Entities `OOV`, `Name` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1678134144791.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_name_vsv_all_901529445_en_4.3.1_3.0_1678134144791.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_name_vsv_all_901529445","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_name_vsv_all_901529445| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/ismail-lucifer011/autotrain-name_vsv_all-901529445 --- layout: model title: Detect Persons, Locations, Organizations and Misc Entities in Italian (WikiNER 840B 300) author: John Snow Labs name: wikiner_840B_300 date: 2020-02-03 task: Named Entity Recognition language: it edition: Spark NLP 2.4.0 spark_version: 2.4 tags: [ner, it, open_source] supported: true annotator: NerDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description WikiNER is a Named Entity Recognition (or NER) model, meaning it annotates text to find features like the names of people, places, and organizations. This NER model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together. WikiNER 840B 300 is trained with GloVe 840B 300 word embeddings, so be sure to use the same embeddings in the pipeline. {:.h2_title} ## Predicted Entities Persons-`PER`, Locations-`LOC`, Organizations-`ORG`, Miscellaneous-`MISC`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_IT){:.button.button-orange}{:target="_blank"} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_IT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_it_2.4.0_2.4_1579699913554.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/wikiner_840B_300_it_2.4.0_2.4_1579699913554.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash +-------------------------------+---------+ |chunk |ner_label| +-------------------------------+---------+ |William Henry Gates III |PER | |Microsoft Corporation |ORG | |Microsoft |ORG | |Gates |PER | |Seattle |LOC | |Washington |LOC | |Gates |PER | |Microsoft |ORG | |Paul Allen |PER | |Albuquerque |LOC | |New Mexico |LOC | |Gates |PER | |CEO |ORG | |CEO |ORG | |Gates |PER | |Gates |PER | |Microsoft |ORG | |Bill & Melinda Gates Foundation|ORG | |Melinda Gates |PER | |Ray Ozzie |PER | +-------------------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|wikiner_840B_300| |Type:|ner| |Compatibility:| Spark NLP 2.4.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|it| |Case sensitive:|false| {:.h2_title} ## Data Source The model is trained based on data from [https://it.wikipedia.org](https://it.wikipedia.org) --- layout: model title: Word2Vec Embeddings in Odia (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, or, open_source] task: Embeddings language: or edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_or_3.4.1_3.0_1647451017345.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_or_3.4.1_3.0_1647451017345.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetectorDL = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en")\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models")\ .setInputCols(["sentence","token"])\ .setOutputCol("embeddings") ner = MedicalNerModel.pretrained('ner_eu_clinical_case', "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = pipeline(stages=[ document_assembler, sentenceDetectorDL, tokenizer, word_embeddings, ner, ner_converter]) data = spark.createDataFrame([["""A 3-year-old boy with autistic disorder on hospital of pediatric ward A at university hospital. He has no family history of illness or autistic spectrum disorder. The child was diagnosed with a severe communication disorder, with social interaction difficulties and sensory processing delay. Blood work was normal (thyroid-stimulating hormone (TSH), hemoglobin, mean corpuscular volume (MCV), and ferritin). Upper endoscopy also showed a submucosal tumor causing subtotal obstruction of the gastric outlet. Because a gastrointestinal stromal tumor was suspected, distal gastrectomy was performed. Histopathological examination revealed spindle cell proliferation in the submucosal layer."""]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentenceDetector = SentenceDetectorDLModel.pretrained() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical","en","clinical/models") .setInputCols(Array("sentence","token")) .setOutputCol("embeddings") val ner_model = MedicalNerModel.pretrained("ner_eu_clinical_case", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documenter, sentenceDetector, tokenizer, word_embeddings, ner_model, ner_converter)) val data = Seq(Array("""A 3-year-old boy with autistic disorder on hospital of pediatric ward A at university hospital. He has no family history of illness or autistic spectrum disorder. The child was diagnosed with a severe communication disorder, with social interaction difficulties and sensory processing delay. Blood work was normal (thyroid-stimulating hormone (TSH), hemoglobin, mean corpuscular volume (MCV), and ferritin). Upper endoscopy also showed a submucosal tumor causing subtotal obstruction of the gastric outlet. Because a gastrointestinal stromal tumor was suspected, distal gastrectomy was performed. Histopathological examination revealed spindle cell proliferation in the submucosal layer.""")).toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.clinical_case_eu").predict("""A 3-year-old boy with autistic disorder on hospital of pediatric ward A at university hospital. He has no family history of illness or autistic spectrum disorder. The child was diagnosed with a severe communication disorder, with social interaction difficulties and sensory processing delay. Blood work was normal (thyroid-stimulating hormone (TSH), hemoglobin, mean corpuscular volume (MCV), and ferritin). Upper endoscopy also showed a submucosal tumor causing subtotal obstruction of the gastric outlet. Because a gastrointestinal stromal tumor was suspected, distal gastrectomy was performed. Histopathological examination revealed spindle cell proliferation in the submucosal layer.""") ```

## Results ```bash +------------------------------+------------------+ |chunk |ner_label | +------------------------------+------------------+ |A 3-year-old boy |patient | |autistic disorder |clinical_condition| |He |patient | |illness |clinical_event | |autistic spectrum disorder |clinical_condition| |The child |patient | |diagnosed |clinical_event | |disorder |clinical_event | |difficulties |clinical_event | |Blood |bodypart | |work |clinical_event | |normal |units_measurements| |hormone |clinical_event | |hemoglobin |clinical_event | |volume |clinical_event | |endoscopy |clinical_event | |showed |clinical_event | |tumor |clinical_condition| |causing |clinical_event | |obstruction |clinical_event | |the gastric outlet |bodypart | |gastrointestinal stromal tumor|clinical_condition| |suspected |clinical_event | |gastrectomy |clinical_event | |examination |clinical_event | |revealed |clinical_event | |spindle cell proliferation |clinical_condition| |the submucosal layer |bodypart | +------------------------------+------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_eu_clinical_case| |Compatibility:|Healthcare NLP 4.2.7+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|849.0 KB| ## References The corpus used for model training is provided by European Clinical Case Corpus (E3C), a project aimed at offering a freely available multilingual corpus of semantically annotated clinical narratives. ## Benchmarking ```bash label tp fp fn total precision recall f1 date_time 54.0 7.0 15.0 69.0 0.8852 0.7826 0.8308 units_measurements 111.0 48.0 12.0 123.0 0.6981 0.9024 0.7872 clinical_condition 93.0 47.0 81.0 174.0 0.6643 0.5345 0.5924 patient 119.0 16.0 5.0 124.0 0.8815 0.9597 0.9189 clinical_event 331.0 126.0 89.0 420.0 0.7243 0.7881 0.7548 bodypart 171.0 58.0 84.0 255.0 0.7467 0.6706 0.7066 macro - - - - - - 0.7651 micro - - - - - - 0.7454 ``` --- layout: model title: Legal Laws Clause Binary Classifier author: John Snow Labs name: legclf_laws_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `laws` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `laws` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_laws_clause_en_1.0.0_3.2_1660123661133.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_laws_clause_en_1.0.0_3.2_1660123661133.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_laws_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[laws]| |[other]| |[other]| |[laws]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_laws_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support compliance-with-laws 0.92 0.94 0.93 64 other 0.97 0.96 0.97 141 accuracy - - 0.96 205 macro-avg 0.95 0.95 0.95 205 weighted-avg 0.96 0.96 0.96 205 ``` --- layout: model title: English BertForQuestionAnswering model (from aodiniz) author: John Snow Labs name: bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert_uncased_L-4_H-768_A-12_cord19-200616_squad2_covid-qna` is a English model orginally trained by `aodiniz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna_en_4.0.0_3.0_1654185337634.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna_en_4.0.0_3.0_1654185337634.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2_covid_cord19.bert.uncased_4l_768d_a12a_768d").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_uncased_L_4_H_768_A_12_cord19_200616_squad2_covid_qna| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|194.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/aodiniz/bert_uncased_L-4_H-768_A-12_cord19-200616_squad2_covid-qna --- layout: model title: English BERT Embeddings Cased model (from mrm8488) author: John Snow Labs name: bert_embeddings_scibert_scivocab_finetuned_cord19 date: 2022-07-15 tags: [en, open_source, bert, embeddings] task: Embeddings language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BERT Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `scibert_scivocab-finetuned-CORD19` is a English model originally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_scibert_scivocab_finetuned_cord19_en_4.0.0_3.0_1657880897640.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_scibert_scivocab_finetuned_cord19_en_4.0.0_3.0_1657880897640.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = BertEmbeddings.pretrained("bert_embeddings_scibert_scivocab_finetuned_cord19","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = BertEmbeddings.pretrained("bert_embeddings_scibert_scivocab_finetuned_cord19","en") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_scibert_scivocab_finetuned_cord19| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|412.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/mrm8488/scibert_scivocab-finetuned-CORD19 --- layout: model title: Named Entity Recognition for Japanese (BertForTokenClassification) author: John Snow Labs name: bert_token_classifier_ner_ud_gsd date: 2021-09-10 tags: [ner, ja, open_source, bert] task: Named Entity Recognition language: ja edition: Spark NLP 3.2.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model annotates named entities in a text, that can be used to find features such as names of people, places, and organizations. It has BERT embeddings integrated with an attached dense layer at the end, to classify tokens directly. Word segmentation is needed to extract the tokens. The model uses BERT embeddings from https://github.com/cl-tohoku/bert-japanese. ## Predicted Entities `ORDINAL`, `PERSON`, `LAW`, `MOVEMENT`, `LOC`, `WORK_OF_ART`, `DATE`, `NORP`, `TITLE_AFFIX`, `QUANTITY`, `FAC`, `TIME`, `MONEY`, `LANGUAGE`, `GPE`, `EVENT`, `ORG`, `PERCENT`, `PRODUCT` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ner_ud_gsd_ja_3.2.2_3.0_1631279615344.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_token_classifier_ner_ud_gsd_ja_3.2.2_3.0_1631279615344.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.base import * from sparknlp.annotator import * from pyspark.ml import Pipeline documentAssembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") sentenceDetector = SentenceDetector() \ .setInputCols("document") \ .setOutputCol("sentence") word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") \ .setInputCols(["sentence"]) \ .setOutputCol("token") nerTagger = BertForTokenClassification \ .pretrained("bert_token_classifier_ner_ud_gsd", "ja") \ .setInputCols(["sentence",'token']) \ .setOutputCol("ner") pipeline = Pipeline().setStages([ documentAssembler, sentenceDetector, word_segmenter, nerTagger ]) data = spark.createDataFrame([["宮本茂氏は、日本の任天堂のゲームプロデューサーです。"]]).toDF("text") model = pipeline.fit(data) result = model.transform(data) ``` ```scala import spark.implicits._ import com.johnsnowlabs.nlp.DocumentAssembler import com.johnsnowlabs.nlp.annotator.{SentenceDetector, WordSegmenterModel} import com.johnsnowlabs.nlp.annotator.BertForTokenClassification import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val word_segmenter = WordSegmenterModel.pretrained("wordseg_gsd_ud", "ja") .setInputCols("sentence") .setOutputCol("token") val nerTagger = BertForTokenClassification .pretrained("bert_token_classifier_ner_ud_gsd", "ja") .setInputCols("sentence", "token") .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array( documentAssembler, sentence, word_segmenter, nerTagger )) val data = Seq("宮本茂氏は、日本の任天堂のゲームプロデューサーです。").toDF("text") val model = pipeline.fit(data) val result = model.transform(data) result.selectExpr("explode(arrays_zip(token.result, ner.result))") .selectExpr("col'0' as token", "col'1' as ner") .show() ``` {:.nlu-block} ```python import nlu nlu.load("ja.classify.token_bert.classifier_ner_ud_gsd").predict("""Put your text here.""") ```

## Results ```bash +--------------+--------+ | token| ner| +--------------+--------+ | 宮本|B-PERSON| | 茂|B-PERSON| | 氏| O| | は| O| | 、| O| | 日本| O| | の| O| | 任天| O| | 堂| O| | の| O| | ゲーム| O| |プロデューサー| O| | です| O| | 。| O| +--------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_ud_gsd| |Compatibility:|Spark NLP 3.2.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[ner]| |Language:|ja| |Case sensitive:|true| |Max sentense length:|128| ## Data Source The model was trained on the Universal Dependencies, curated by Google. A NER version was created by megagonlabs: https://github.com/megagonlabs/UD_Japanese-GSD Reference: Asahara, M., Kanayama, H., Tanaka, T., Miyao, Y., Uematsu, S., Mori, S., Matsumoto, Y., Omura, M., & Murawaki, Y. (2018). Universal Dependencies Version 2 for Japanese. In LREC-2018. ## Benchmarking ```bash label precision recall f1-score support DATE 0.87 0.94 0.91 191 EVENT 0.27 0.82 0.41 17 FAC 0.66 0.63 0.64 62 GPE 0.40 0.59 0.48 70 LANGUAGE 0.50 0.67 0.57 6 LAW 0.00 0.00 0.00 0 LOC 0.59 0.69 0.63 35 MONEY 0.90 0.90 0.90 20 MOVEMENT 0.27 1.00 0.43 3 NORP 0.46 0.57 0.50 46 O 0.98 0.97 0.98 11897 ORDINAL 0.94 0.70 0.80 43 ORG 0.49 0.43 0.46 204 PERCENT 1.00 0.80 0.89 20 PERSON 0.56 0.68 0.61 105 PRODUCT 0.32 0.53 0.40 30 QUANTITY 0.83 0.75 0.79 189 TIME 0.88 0.64 0.74 44 TITLE_AFFIX 0.33 0.80 0.47 10 WORK_OF_ART 0.67 0.76 0.71 42 accuracy - - 0.95 13034 macro-avg 0.60 0.69 0.62 13034 weighted-avg 0.96 0.95 0.95 13034 ``` --- layout: model title: Legal Investments Clause Binary Classifier author: John Snow Labs name: legclf_investments_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `investments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `investments` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_investments_clause_en_1.0.0_3.2_1660123646127.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_investments_clause_en_1.0.0_3.2_1660123646127.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_investments_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[investments]| |[other]| |[other]| |[investments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_investments_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support investments 0.97 0.85 0.91 34 other 0.94 0.99 0.96 80 accuracy - - 0.95 114 macro-avg 0.95 0.92 0.93 114 weighted-avg 0.95 0.95 0.95 114 ``` --- layout: model title: Pipeline to Detect Living Species author: John Snow Labs name: bert_token_classifier_ner_living_species_pipeline date: 2023-03-20 tags: [en, ner, clinical, licensed, bertfortokenclassification] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_living_species](https://nlp.johnsnowlabs.com/2022/06/26/bert_token_classifier_ner_living_species_en_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_pipeline_en_4.3.0_3.2_1679304760714.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_living_species_pipeline_en_4.3.0_3.2_1679304760714.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("bert_token_classifier_ner_living_species_pipeline", "en", "clinical/models") text = '''42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("bert_token_classifier_ner_living_species_pipeline", "en", "clinical/models") val text = "42-year-old woman with end-stage chronic kidney disease, secondary to lupus nephropathy, and on peritoneal dialysis. History of four episodes of bacterial peritonitis and change of Tenckhoff catheter six months prior to admission due to catheter dysfunction. Three peritoneal fluid samples during her hospitalisation tested positive for Fusarium spp. The patient responded favourably and continued outpatient treatment with voriconazole (4mg/kg every 12 hours orally). All three isolates were identified as species of the Fusarium solani complex. In vitro susceptibility to itraconazole, voriconazole and posaconazole, according to Clinical and Laboratory Standards Institute - CLSI (M38-A) methodology, showed a minimum inhibitory concentration (MIC) in all three isolates and for all three antifungals of >16 μg/mL." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:------------------------|--------:|------:|:------------|-------------:| | 0 | woman | 12 | 16 | HUMAN | 0.986743 | | 1 | bacterial | 145 | 153 | SPECIES | 0.975256 | | 2 | Fusarium spp | 337 | 348 | SPECIES | 0.998142 | | 3 | patient | 355 | 361 | HUMAN | 0.994012 | | 4 | species | 507 | 513 | SPECIES | 0.962562 | | 5 | Fusarium solani complex | 522 | 544 | SPECIES | 0.999028 | | 6 | antifungals | 792 | 802 | SPECIES | 0.999852 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_token_classifier_ner_living_species_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|404.8 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - MedicalBertForTokenClassifier - NerConverterInternalModel --- layout: model title: Translate English to Macedonian Pipeline author: John Snow Labs name: translate_en_mk date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, mk, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `mk` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_mk_xx_2.7.0_2.4_1609687312775.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_mk_xx_2.7.0_2.4_1609687312775.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_mk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_mk", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.mk').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_mk| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Stopwords Remover for Danish language (219 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, da, open_source] task: Stop Words Removal language: da edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_da_3.4.1_3.0_1646672996410.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_da_3.4.1_3.0_1646672996410.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result | +-------+ |[bedre]| +-------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|da| |Size:|2.0 KB| --- layout: model title: Pipeline to Detect Clinical Entities (ner_jsl_slim) author: John Snow Labs name: ner_jsl_slim_pipeline date: 2023-03-07 tags: [ner, en, clinical, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.1 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_jsl_slim](https://nlp.johnsnowlabs.com/2021/08/13/ner_jsl_slim_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_jsl_slim_pipeline_en_4.3.1_3.2_1678195679312.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_jsl_slim_pipeline_en_4.3.1_3.2_1678195679312.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_jsl_slim_pipeline", "en", "clinical/models") text = "Hyperparathyroidism was considered upon the fourth occasion. The history of weakness and generalized joint pains were present. He also had history of epigastric pain diagnosed informally as gastritis. He had previously had open reduction and internal fixation for the initial two fractures under general anesthesia. He sustained mandibular fracture." result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_jsl_slim_pipeline", "en", "clinical/models") val text = "Hyperparathyroidism was considered upon the fourth occasion. The history of weakness and generalized joint pains were present. He also had history of epigastric pain diagnosed informally as gastritis. He had previously had open reduction and internal fixation for the initial two fractures under general anesthesia. He sustained mandibular fracture." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.jsl_slim.pipeline").predict("""Hyperparathyroidism was considered upon the fourth occasion. The history of weakness and generalized joint pains were present. He also had history of epigastric pain diagnosed informally as gastritis. He had previously had open reduction and internal fixation for the initial two fractures under general anesthesia. He sustained mandibular fracture.""") ```

## Results ```bash | | chunks | begin | end | entities | confidence | |---:|:-------------------------------------|--------:|------:|:--------------------------|-------------:| | 0 | Hyperparathyroidism | 0 | 18 | Disease_Syndrome_Disorder | 0.9977 | | 1 | weakness | 76 | 83 | Symptom | 0.9744 | | 2 | generalized joint pains | 89 | 111 | Symptom | 0.584067 | | 3 | He | 127 | 128 | Demographics | 0.9996 | | 4 | epigastric pain | 150 | 164 | Symptom | 0.66655 | | 5 | gastritis | 190 | 198 | Disease_Syndrome_Disorder | 0.9874 | | 6 | He | 201 | 202 | Demographics | 0.9995 | | 7 | open reduction and internal fixation | 223 | 258 | Procedure | 0.61648 | | 8 | fractures under general anesthesia | 280 | 313 | Drug | 0.79585 | | 9 | He | 316 | 317 | Demographics | 0.9992 | | 10 | sustained mandibular fracture | 319 | 347 | Disease_Syndrome_Disorder | 0.662467 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_jsl_slim_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Pipeline to Mapping RxNorm Codes with Corresponding National Drug Codes (NDC) author: John Snow Labs name: rxnorm_ndc_mapping date: 2023-06-13 tags: [en, licensed, clinical, pipeline, chunk_mapping, rxnorm, ndc] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps RXNORM codes to NDC codes without using any text data. You’ll just feed white space-delimited RXNORM codes and it will return the corresponding two different types of ndc codes which are called `package ndc` and `product ndc`. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapping_en_4.4.4_3.2_1686665536839.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapping_en_4.4.4_3.2_1686665536839.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("rxnorm_ndc_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(1652674 259934) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("rxnorm_ndc_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(1652674 259934) ``` {:.nlu-block} ```python import nlu nlu.load("en.map_entity.rxnorm_to_ndc.pipe").predict("""Put your text here.""") ```

## Results ```bash Results {'document': ['1652674 259934'], 'package_ndc': ['62135-0625-60', '13349-0010-39'], 'product_ndc': ['46708-0499', '13349-0010'], 'rxnorm_code': ['1652674', '259934']} {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_ndc_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|4.0 MB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel - ChunkMapperModel --- layout: model title: Legal Interest Clause Binary Classifier author: John Snow Labs name: legclf_interest_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `interest` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `interest` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_interest_clause_en_1.0.0_3.2_1660122572067.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_interest_clause_en_1.0.0_3.2_1660122572067.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_interest_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[interest]| |[other]| |[other]| |[interest]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_interest_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support conflict-of-interest 0.90 0.81 0.85 32 other 0.95 0.97 0.96 108 accuracy - - 0.94 140 macro-avg 0.92 0.89 0.91 140 weighted-avg 0.93 0.94 0.93 140 ``` --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from JoanTirant) author: John Snow Labs name: roberta_qa_joantirant_base_bne_finetuned_s_c date: 2023-01-20 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-finetuned-sqac` is a Spanish model originally trained by `JoanTirant`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_joantirant_base_bne_finetuned_s_c_es_4.3.0_3.0_1674212952508.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_joantirant_base_bne_finetuned_s_c_es_4.3.0_3.0_1674212952508.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_joantirant_base_bne_finetuned_s_c","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_joantirant_base_bne_finetuned_s_c","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_joantirant_base_bne_finetuned_s_c| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.4 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/JoanTirant/roberta-base-bne-finetuned-sqac --- layout: model title: English asr_iloko TFWav2Vec2ForCTC from denden author: John Snow Labs name: pipeline_asr_iloko date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_iloko` is a English model originally trained by denden. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_iloko_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_iloko_en_4.2.0_3.0_1664095414916.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_iloko_en_4.2.0_3.0_1664095414916.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_iloko', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_iloko", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_iloko| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Jurisdiction Clause Binary Classifier author: John Snow Labs name: legclf_jurisdiction_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `jurisdiction` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `jurisdiction` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_jurisdiction_clause_en_1.0.0_3.2_1660122594462.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_jurisdiction_clause_en_1.0.0_3.2_1660122594462.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_jurisdiction_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[jurisdiction]| |[other]| |[other]| |[jurisdiction]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_jurisdiction_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support jurisdiction 0.94 0.96 0.95 70 other 0.99 0.98 0.98 208 accuracy - - 0.97 278 macro-avg 0.96 0.97 0.97 278 weighted-avg 0.97 0.97 0.97 278 ``` --- layout: model title: English BertForQuestionAnswering model (from jatinshah) author: John Snow Labs name: bert_qa_jatinshah_bert_finetuned_squad date: 2022-06-06 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad` is a English model orginally trained by `jatinshah`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_jatinshah_bert_finetuned_squad_en_4.0.0_3.0_1654535654266.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_jatinshah_bert_finetuned_squad_en_4.0.0_3.0_1654535654266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_jatinshah_bert_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_jatinshah_bert_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_jatinshah").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_jatinshah_bert_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/jatinshah/bert-finetuned-squad --- layout: model title: Spanish Named Entity Recognition (from lirondos) author: John Snow Labs name: bert_ner_anglicisms_spanish_mbert date: 2022-05-09 tags: [bert, ner, token_classification, es, open_source] task: Named Entity Recognition language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, uploaded to Hugging Face, adapted and imported into Spark NLP. `anglicisms-spanish-mbert` is a Spanish model orginally trained by `lirondos`. ## Predicted Entities `OTHER`, `ENG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_ner_anglicisms_spanish_mbert_es_3.4.2_3.0_1652096511636.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_ner_anglicisms_spanish_mbert_es_3.4.2_3.0_1652096511636.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_ner_anglicisms_spanish_mbert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|es| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/lirondos/anglicisms-spanish-mbert - https://github.com/lirondos/coalas/ - https://github.com/lirondos/coalas/ - https://github.com/lirondos/coalas/ - https://arxiv.org/abs/2203.16169 --- layout: model title: Multilabel Classification of NDA Clauses (sentences, medium) author: John Snow Labs name: legmulticlf_mnda_sections_other date: 2023-02-09 tags: [mnda, nda, en, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: MultiClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This models is a version of `legmulticlf_mnda_sections` (medium) but including more negative examples (OTHER) to reinforce difference between groups and returning `OTHER` also as synonym to `[]`. It should be run on sentences of the NDA clauses, and will retrieve a series of 1..N labels for each of them. The possible clause types detected my this model in NDA / MNDA aggrements are: 1. Parties to the Agreement - Names of the Parties Clause 2. Identification of What Information Is Confidential - Definition of Confidential Information Clause 3. Use of Confidential Information: Permitted Use Clause and Obligations of the Recipient 4. Time Frame of the Agreement - Termination Clause 5. Return of Confidential Information Clause 6. Remedies for Breaches of Agreement - Remedies Clause 7. Non-Solicitation Clause 8. Dispute Resolution Clause 9. Exceptions Clause 10. Non-competition clause 11. Other: Nothing of the above (synonym to `[]`)- ## Predicted Entities `APPLIC_LAW`, `ASSIGNMENT`, `DEF_OF_CONF_INFO`, `DISPUTE_RESOL`, `EXCEPTIONS`, `NAMES_OF_PARTIES`, `NON_COMP`, `NON_SOLIC`, `PREAMBLE`, `REMEDIES`, `REQ_DISCL`, `RETURN_OF_CONF_INFO`, `TERMINATION`, `USE_OF_CONF_INFO`, `OTHER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legmulticlf_mnda_sections_other_en_1.0.0_3.0_1675938628942.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legmulticlf_mnda_sections_other_en_1.0.0_3.0_1675938628942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = ( nlp.DocumentAssembler().setInputCol("text").setOutputCol("document") ) sentence_splitter = ( nlp.SentenceDetector() .setInputCols(["document"]) .setOutputCol("sentence") .setCustomBounds(["\n"]) ) embeddings = ( nlp.UniversalSentenceEncoder.pretrained() .setInputCols("document") .setOutputCol("sentence_embeddings") ) classsifierdl_pred = nlp.MultiClassifierDLModel.pretrained('legmulticlf_mnda_sections_other', 'en', 'legal/models')\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("class") clf_pipeline = nlp.Pipeline(stages=[document_assembler,embeddings, classsifierdl_pred]) df = spark.createDataFrame([["Governing Law.\nThis Agreement shall be govern..."]]).toDF("text") res = clf_pipeline.fit(df).transform(df) res.select('text', 'class.result').show() res.select('class.result') ```

## Results ```bash [APPLIC_LAW] Governing Law.\nThis Agreement shall be govern... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legmulticlf_mnda_sections_other| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|13.0 MB| ## References In-house MNDA ## Benchmarking ```bash label precision recall f1-score support APPLIC_LAW 0.90 0.86 0.88 22 ASSIGNMENT 0.90 0.90 0.90 21 DEF_OF_CONF_INFO 0.83 0.80 0.82 25 DISPUTE_RESOL 0.84 0.75 0.79 36 EXCEPTIONS 0.94 0.85 0.89 20 NAMES_OF_PARTIES 0.85 0.95 0.90 37 NON_COMP 0.89 0.94 0.92 18 NON_SOLIC 0.90 1.00 0.95 9 OTHER 0.95 0.90 0.92 123 PREAMBLE 0.88 0.81 0.84 36 REMEDIES 0.74 0.74 0.74 27 REQ_DISCL 0.86 0.75 0.80 16 RETURN_OF_CONF_INFO 0.85 0.88 0.87 26 TERMINATION 0.79 0.79 0.79 19 USE_OF_CONF_INFO 0.79 0.71 0.75 31 micro-avg 0.88 0.85 0.86 466 macro-avg 0.86 0.84 0.85 466 weighted-avg 0.88 0.85 0.86 466 samples-avg 0.83 0.85 0.83 466 ``` --- layout: model title: Pipeline to Detect clinical events (biobert) author: John Snow Labs name: ner_events_biobert_pipeline date: 2023-03-20 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_events_biobert](https://nlp.johnsnowlabs.com/2021/04/01/ner_events_biobert_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_biobert_pipeline_en_4.3.0_3.2_1679315540096.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_biobert_pipeline_en_4.3.0_3.2_1679315540096.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_events_biobert_pipeline", "en", "clinical/models") text = '''The patient presented to the emergency room last evening.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_events_biobert_pipeline", "en", "clinical/models") val text = "The patient presented to the emergency room last evening." val result = pipeline.fullAnnotate(text) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.biobert_events.pipeline").predict("""The patient presented to the emergency room last evening.""") ```

## Results ```bash | | ner_chunk | begin | end | ner_label | confidence | |---:|:-------------------|--------:|------:|:--------------|-------------:| | 0 | presented | 12 | 20 | OCCURRENCE | 0.5019 | | 1 | the emergency room | 25 | 42 | CLINICAL_DEPT | 0.695333 | | 2 | last evening | 44 | 55 | DATE | 0.7621 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_events_biobert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|422.1 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Mapping RxNorm Codes with Corresponding National Drug Codes author: John Snow Labs name: rxnorm_ndc_mapper date: 2022-05-20 tags: [ndc, rxnorm, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps RxNorm and RxNorm Extension codes with corresponding National Drug Codes (NDC). ## Predicted Entities `Product NDC`, `Package NDC` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapper_en_3.5.1_3.0_1653061064192.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapper_en_3.5.1_3.0_1653061064192.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('ner_chunk') sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_base_cased_mli', 'en','clinical/models') .setInputCols(["ner_chunk"]) .setOutputCol("sentence_embeddings") .setCaseSensitive(False) rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") .setInputCols(["sentence_embeddings"]) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") chunkerMapper_product = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")) .setInputCols(["rxnorm_code"]) .setOutputCol("Product NDC") .setRel("Product NDC") chunkerMapper_package = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")) .setInputCols(["rxnorm_code"]) .setOutputCol("Package NDC") .setRel("Package NDC") pipeline = Pipeline().setStages([document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_product, chunkerMapper_package ]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) result = lp.annotate(['doxycycline hyclate 50 MG Oral Tablet', 'macadamia nut 100 MG/ML']) ``` ```scala val document_assembler = DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sentence_embeddings") .setCaseSensitive(False) val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") .setInputCols(Array("sentence_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper_product = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")) .setInputCols(Array("rxnorm_code")) .setOutputCol("Product NDC") .setRel("Product NDC") val chunkerMapper_package = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")) .setInputCols(Array("rxnorm_code")) .setOutputCol("Package NDC") .setRel("Package NDC") val pipeline = Pipeline().setStages(Array(document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_product, chunkerMapper_package )) val text_data = Seq("doxycycline hyclate 50 MG Oral Tablet'", "macadamia nut 100 MG/ML").toDF("text") val res = pipeline.fit(text_data).transform(text_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.rxnorm_to_ndc").predict("""Product NDC""") ```

## Results ```bash | | ner_chunk | rxnorm_code | Package NDC | Product NDC | |---:|:------------------------------------------|:--------------|:------------------|:---------------| | 0 | ['doxycycline hyclate 50 MG Oral Tablet'] | ['1652674'] | ['62135-0625-60'] | ['62135-0625'] | | 1 | ['macadamia nut 100 MG/ML'] | ['212433'] | ['00187-1474-08'] | ['00187-1474'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_ndc_mapper| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|2.0 MB| --- layout: model title: Detect Chemicals in Text (embeddings_clinical_large) author: John Snow Labs name: ner_chemicals_emb_clinical_large date: 2023-06-02 tags: [ner, clinical, licensed, en, chemicals] task: Named Entity Recognition language: en edition: Healthcare NLP 4.4.3 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Extract different types of chemical compounds mentioned in text using pretrained NER model. ( Trained with embeddings_clinical_large ) ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_CHEMICALS/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemicals_emb_clinical_large_en_4.4.3_3.0_1685713522857.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemicals_emb_clinical_large_en_4.4.3_3.0_1685713522857.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | chunks | begin | end | entities | |---:|:-------------------------------------------------|--------:|------:|:-----------| | 0 | resveratrol | 48 | 58 | CHEM | | 1 | trans - 3, 5, 4 - trihydroxystilbene) glucosides | 61 | 108 | CHEM | | 2 | Resveratrol | 136 | 146 | CHEM | | 3 | trans - 3, 5, 4 - trihydroxystilbene | 149 | 185 | CHEM | | 4 | RSV | 189 | 191 | CHEM | | 5 | polyphenol | 206 | 215 | CHEM | | 6 | NAD(+) | 300 | 305 | CHEM | | 7 | superoxide | 436 | 445 | CHEM | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemicals_emb_clinical_large| |Compatibility:|Healthcare NLP 4.4.3+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token, embeddings]| |Output Labels:|[ner]| |Language:|en| |Size:|2.8 MB| ## Benchmarking ```bash label precision recall f1-score support CHEM 0.94 0.93 0.94 62001 micro-avg 0.94 0.93 0.94 62001 macro-avg 0.94 0.93 0.94 62001 weighted-avg 0.94 0.93 0.94 62001 ``` --- layout: model title: English RobertaForQuestionAnswering (from AnonymousSub) author: John Snow Labs name: roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739809828.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0_en_4.0.0_3.0_1655739809828.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squadv2.roberta.base_rule_based_twostagetriplet_hier_epochs_1_shard_1.by_AnonymousSub").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|306.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/AnonymousSub/rule_based_roberta_twostagetriplet_hier_epochs_1_shard_1_squad2.0 --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from machine2049) author: John Snow Labs name: distilbert_qa_machine2049_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `machine2049`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_machine2049_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772141683.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_machine2049_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772141683.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_machine2049_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_machine2049_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_machine2049_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/machine2049/distilbert-base-uncased-finetuned-squad --- layout: model title: Pipeline to Extract textual entities in biomedical texts author: John Snow Labs name: ner_nature_nero_clinical_pipeline date: 2023-03-14 tags: [ner, en, clinical, licensed] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_nature_nero_clinical](https://nlp.johnsnowlabs.com/2022/02/08/ner_nature_nero_clinical_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_nature_nero_clinical_pipeline_en_4.3.0_3.2_1678776843378.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_nature_nero_clinical_pipeline_en_4.3.0_3.2_1678776843378.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_nature_nero_clinical_pipeline", "en", "clinical/models") text = '''he patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_nature_nero_clinical_pipeline", "en", "clinical/models") val text = "he patient is a 21-day-old Caucasian male here for 2 days of congestion - mom has been suctioning yellow discharge from the patient's nares, plus she has noticed some mild problems with his breathing while feeding (but negative for any perioral cyanosis or retractions). One day ago, mom also noticed a tactile temperature and gave the patient Tylenol. Baby also has had some decreased p.o. intake. His normal breast-feeding is down from 20 minutes q.2h. to 5 to 10 minutes secondary to his respiratory congestion. He sleeps well, but has been more tired and has been fussy over the past 2 days. The parents noticed no improvement with albuterol treatments given in the ER. His urine output has also decreased; normally he has 8 to 10 wet and 5 dirty diapers per 24 hours, now he has down to 4 wet diapers per 24 hours. Mom denies any diarrhea. His bowel movements are yellow colored and soft in nature." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:---------------------------------------------|--------:|------:|:----------------------|-------------:| | 0 | perioral cyanosis | 236 | 252 | Medicalfinding | 0.198 | | 1 | One day | 271 | 277 | Duration | 0.35005 | | 2 | mom | 284 | 286 | Namedentity | 0.1301 | | 3 | tactile temperature | 303 | 321 | Quantityormeasurement | 0.1074 | | 4 | patient Tylenol | 336 | 350 | Chemical | 0.20805 | | 5 | decreased p.o. intake | 376 | 396 | Medicalprocedure | 0.105725 | | 6 | normal breast-feeding | 403 | 423 | Medicalfinding | 0.1769 | | 7 | 20 minutes q.2h | 438 | 452 | Timepoint | 0.275333 | | 8 | 5 to 10 minutes | 458 | 472 | Duration | 0.22645 | | 9 | respiratory congestion | 491 | 512 | Medicalfinding | 0.1423 | | 10 | past 2 days | 583 | 593 | Duration | 0.256867 | | 11 | parents | 600 | 606 | Persongroup | 0.9441 | | 12 | improvement | 619 | 629 | Process | 0.147 | | 13 | albuterol treatments | 636 | 655 | Medicalprocedure | 0.305 | | 14 | ER | 670 | 671 | Bodypart | 0.2024 | | 15 | urine output | 678 | 689 | Quantityormeasurement | 0.1283 | | 16 | 8 to 10 wet and 5 dirty diapers per 24 hours | 727 | 770 | Measurement | 0.121327 | | 17 | 4 wet diapers per 24 hours | 792 | 817 | Measurement | 0.1611 | | 18 | Mom | 820 | 822 | Person | 0.9515 | | 19 | diarrhea | 835 | 842 | Medicalfinding | 0.533 | | 20 | bowel movements | 849 | 863 | Biologicalprocess | 0.2036 | | 21 | soft in nature | 888 | 901 | Biologicalprocess | 0.170467 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_nature_nero_clinical_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: Pipeline to Detect Entities Related to Cancer Therapies author: John Snow Labs name: ner_oncology_therapy_pipeline date: 2023-03-09 tags: [clinical, en, licensed, oncology, treatment, ner] task: Named Entity Recognition language: en edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_oncology_therapy](https://nlp.johnsnowlabs.com/2022/11/24/ner_oncology_therapy_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_pipeline_en_4.3.0_3.2_1678351787302.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_oncology_therapy_pipeline_en_4.3.0_3.2_1678351787302.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_oncology_therapy_pipeline", "en", "clinical/models") text = '''The had previously undergone a left mastectomy and an axillary lymph node dissection for a left breast cancer twenty years ago. The tumor was positive for ER and PR. Postoperatively, radiotherapy was administered to her breast. The cancer recurred as a right lung metastasis 13 years later. The patient underwent a regimen consisting of adriamycin (60 mg/m2) and cyclophosphamide (600 mg/m2) over six courses, as first line therapy.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_oncology_therapy_pipeline", "en", "clinical/models") val text = "The had previously undergone a left mastectomy and an axillary lymph node dissection for a left breast cancer twenty years ago. The tumor was positive for ER and PR. Postoperatively, radiotherapy was administered to her breast. The cancer recurred as a right lung metastasis 13 years later. The patient underwent a regimen consisting of adriamycin (60 mg/m2) and cyclophosphamide (600 mg/m2) over six courses, as first line therapy." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:-------------------------------|--------:|------:|:----------------------|-------------:| | 0 | mastectomy | 36 | 45 | Cancer_Surgery | 0.9817 | | 1 | axillary lymph node dissection | 54 | 83 | Cancer_Surgery | 0.719725 | | 2 | radiotherapy | 183 | 194 | Radiotherapy | 0.9984 | | 3 | recurred | 239 | 246 | Response_To_Treatment | 0.9481 | | 4 | adriamycin | 337 | 346 | Chemotherapy | 0.9981 | | 5 | 60 mg/m2 | 349 | 356 | Dosage | 0.58815 | | 6 | cyclophosphamide | 363 | 378 | Chemotherapy | 0.9976 | | 7 | 600 mg/m2 | 381 | 389 | Dosage | 0.64205 | | 8 | six courses | 397 | 407 | Cycle_Count | 0.46815 | | 9 | first line | 413 | 422 | Line_Of_Therapy | 0.95015 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_oncology_therapy_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverterInternalModel --- layout: model title: English RoBERTa Embeddings (Large, Biology/Medical) author: John Snow Labs name: roberta_embeddings_pmc_med_bio_mlm_roberta_large date: 2022-04-14 tags: [roberta, embeddings, en, open_source] task: Embeddings language: en nav_key: models edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `pmc-med-bio-mlm-roberta-large` is a English model orginally trained by `raynardj`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_pmc_med_bio_mlm_roberta_large_en_3.4.2_3.0_1649946971676.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_pmc_med_bio_mlm_roberta_large_en_3.4.2_3.0_1649946971676.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_pmc_med_bio_mlm_roberta_large| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| ## References - https://huggingface.co/raynardj/pmc-med-bio-mlm-roberta-large --- layout: model title: Pipeline to Detect Living Species (bert_embeddings_bert_base_fr_cased) author: John Snow Labs name: ner_living_species_bert_pipeline date: 2023-03-13 tags: [fr, ner, clinical, licensed, bert] task: Named Entity Recognition language: fr edition: Healthcare NLP 4.3.0 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_living_species_bert](https://nlp.johnsnowlabs.com/2022/06/23/ner_living_species_bert_fr_3_0.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_pipeline_fr_4.3.0_3.2_1678730531080.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_living_species_bert_pipeline_fr_4.3.0_3.2_1678730531080.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("ner_living_species_bert_pipeline", "fr", "clinical/models") text = '''Femme de 47 ans allergique à l'iode, fumeuse sociale, opérée pour des varices, deux césariennes et un abcès fessier. Vit avec son mari et ses trois enfants, travaille comme enseignante. Initialement, le patient a eu une bonne évolution, mais au 2ème jour postopératoire, il a commencé à montrer une instabilité hémodynamique. Les sérologies pour Coxiella burnetii, Bartonella henselae, Borrelia burgdorferi, Entamoeba histolytica, Toxoplasma gondii, herpès simplex virus 1 et 2, cytomégalovirus, virus d'Epstein Barr, virus de la varicelle et du zona et parvovirus B19 étaient négatives. Cependant, un test au rose Bengale positif pour Brucella, le test de Coombs et les agglutinations étaient également positifs avec un titre de 1/40.''' result = pipeline.fullAnnotate(text) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("ner_living_species_bert_pipeline", "fr", "clinical/models") val text = "Femme de 47 ans allergique à l'iode, fumeuse sociale, opérée pour des varices, deux césariennes et un abcès fessier. Vit avec son mari et ses trois enfants, travaille comme enseignante. Initialement, le patient a eu une bonne évolution, mais au 2ème jour postopératoire, il a commencé à montrer une instabilité hémodynamique. Les sérologies pour Coxiella burnetii, Bartonella henselae, Borrelia burgdorferi, Entamoeba histolytica, Toxoplasma gondii, herpès simplex virus 1 et 2, cytomégalovirus, virus d'Epstein Barr, virus de la varicelle et du zona et parvovirus B19 étaient négatives. Cependant, un test au rose Bengale positif pour Brucella, le test de Coombs et les agglutinations étaient également positifs avec un titre de 1/40." val result = pipeline.fullAnnotate(text) ```

## Results ```bash | | ner_chunks | begin | end | ner_label | confidence | |---:|:---------------------------------|--------:|------:|:------------|-------------:| | 0 | Femme | 0 | 4 | HUMAN | 1 | | 1 | mari | 130 | 133 | HUMAN | 1 | | 2 | enfants | 148 | 154 | HUMAN | 0.9999 | | 3 | patient | 203 | 209 | HUMAN | 0.9993 | | 4 | Coxiella burnetii | 346 | 362 | SPECIES | 0.9879 | | 5 | Bartonella henselae | 365 | 383 | SPECIES | 0.9926 | | 6 | Borrelia burgdorferi | 386 | 405 | SPECIES | 0.9959 | | 7 | Entamoeba histolytica | 408 | 428 | SPECIES | 0.9913 | | 8 | Toxoplasma gondii | 431 | 447 | SPECIES | 0.97845 | | 9 | cytomégalovirus | 479 | 493 | SPECIES | 0.9976 | | 10 | virus d'Epstein Barr | 496 | 515 | SPECIES | 0.967967 | | 11 | virus de la varicelle et du zona | 518 | 549 | SPECIES | 0.985429 | | 12 | parvovirus B19 | 554 | 567 | SPECIES | 0.98595 | | 13 | Brucella | 636 | 643 | SPECIES | 0.9995 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_living_species_bert_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Language:|fr| |Size:|410.6 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverterInternalModel --- layout: model title: Turkish Named Entity Recognition (from akdeniz27) author: John Snow Labs name: xlmroberta_ner_xlm_roberta_base_turkish_ner date: 2022-05-17 tags: [xlm_roberta, ner, token_classification, tr, open_source] task: Named Entity Recognition language: tr edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Named Entity Recognition model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-turkish-ner` is a Turkish model orginally trained by `akdeniz27`. ## Predicted Entities `LOC`, `ORG`, `PER` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_turkish_ner_tr_3.4.2_3.0_1652807661266.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_xlm_roberta_base_turkish_ner_tr_3.4.2_3.0_1652807661266.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_xlm_roberta_base_turkish_ner| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|tr| |Size:|851.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/akdeniz27/xlm-roberta-base-turkish-ner - https://github.com/stefan-it/turkish-bert/files/4558187/nerdata.txt --- layout: model title: Extract relations between drugs and proteins (ReDL) author: John Snow Labs name: redl_drugprot_biobert date: 2022-01-05 tags: [relation_extraction, clinical, en, licensed] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.3.4 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Detect interactions between chemical compounds/drugs and genes/proteins using BERT by classifying whether a specified semantic relation holds between a chemical and gene entities within a sentence or document. The entity labels used during training were derived from the [custom NER model](https://nlp.johnsnowlabs.com/2021/12/20/ner_drugprot_clinical_en.html) created by our team for the [DrugProt corpus](https://zenodo.org/record/5119892). These include `CHEMICAL` for chemical compounds/drugs, `GENE` for genes/proteins and `GENE_AND_CHEMICAL` for entity mentions of type `GENE` and of type `CHEMICAL` that overlap (such as enzymes and small peptides). The relation categories from the [DrugProt corpus](https://zenodo.org/record/5119892) were condensed from 13 categories to 10 categories due to low numbers of examples for certain categories. This merging process involved grouping the `SUBSTRATE_PRODUCT-OF` and `SUBSTRATE` relation categories together and grouping the `AGONIST-ACTIVATOR`, `AGONIST-INHIBITOR` and `AGONIST` relation categories together. ## Predicted Entities `INHIBITOR`, `DIRECT-REGULATOR`, `SUBSTRATE`, `ACTIVATOR`, `INDIRECT-UPREGULATOR`, `INDIRECT-DOWNREGULATOR`, `ANTAGONIST`, `PRODUCT-OF`, `PART-OF`, `AGONIST` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_drugprot_biobert_en_3.3.4_3.0_1641393971428.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_drugprot_biobert_en_3.3.4_3.0_1641393971428.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use In the table below, `redl_drugprot_biobert` RE model, its labels, optimal NER model, and meaningful relation pairs are illustrated. | RE MODEL | RE MODEL LABES | NER MODEL | RE PAIRS | |:---------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:---------------------:|------------------------------------------------------------------------------------| | redl_drugprot_biobert | INHIBITOR,
DIRECT-REGULATOR,
SUBSTRATE,
ACTIVATOR,
INDIRECT-UPREGULATOR,
INDIRECT-DOWNREGULATOR,
ANTAGONIST,
PRODUCT-OF,
PART-OF,
AGONIST | ner_drugprot_clinical | [“checmical-gene”,
“chemical-gene_and_chemical”,
“gene_and_chemical-gene”] |

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentencer = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentences") tokenizer = Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") words_embedder = WordEmbeddingsModel()\ .pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("embeddings") drugprot_ner_tagger = MedicalNerModel.pretrained("ner_drugprot_clinical", "en", "clinical/models")\ .setInputCols("sentences", "tokens", "embeddings")\ .setOutputCol("ner_tags") ner_converter = NerConverter()\ .setInputCols(["sentences", "tokens", "ner_tags"])\ .setOutputCol("ner_chunks") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models")\ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") dependency_parser = DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") # Set a filter on pairs of named entities which will be treated as relation candidates drugprot_re_ner_chunk_filter = RENerChunksFilter()\ .setInputCols(["ner_chunks", "dependencies"])\ .setOutputCol("re_ner_chunks")\ .setMaxSyntacticDistance(4) # .setRelationPairs(['CHEMICAL-GENE']) drugprot_re_Model = RelationExtractionDLModel()\ .pretrained('redl_drugprot_biobert', "en", "clinical/models")\ .setPredictionThreshold(0.9)\ .setInputCols(["re_ner_chunks", "sentences"])\ .setOutputCol("relations") pipeline = Pipeline(stages=[documenter, sentencer, tokenizer, words_embedder, drugprot_ner_tagger, ner_converter, pos_tagger, dependency_parser, drugprot_re_ner_chunk_filter, drugprot_re_Model]) text='''Lipid specific activation of the murine P4-ATPase Atp8a1 (ATPase II). The asymmetric transbilayer distribution of phosphatidylserine (PS) in the mammalian plasma membrane and secretory vesicles is maintained, in part, by an ATP-dependent transporter. This aminophospholipid "flippase" selectively transports PS to the cytosolic leaflet of the bilayer and is sensitive to vanadate, Ca(2+), and modification by sulfhydryl reagents. Although the flippase has not been positively identified, a subfamily of P-type ATPases has been proposed to function as transporters of amphipaths, including PS and other phospholipids. A candidate PS flippase ATP8A1 (ATPase II), originally isolated from bovine secretory vesicles, is a member of this subfamily based on sequence homology to the founding member of the subfamily, the yeast protein Drs2, which has been linked to ribosomal assembly, the formation of Golgi-coated vesicles, and the maintenance of PS asymmetry. To determine if ATP8A1 has biochemical characteristics consistent with a PS flippase, a murine homologue of this enzyme was expressed in insect cells and purified. The purified Atp8a1 is inactive in detergent micelles or in micelles containing phosphatidylcholine, phosphatidic acid, or phosphatidylinositol, is minimally activated by phosphatidylglycerol or phosphatidylethanolamine (PE), and is maximally activated by PS. The selectivity for PS is dependent upon multiple elements of the lipid structure. Similar to the plasma membrane PS transporter, Atp8a1 is activated only by the naturally occurring sn-1,2-glycerol isomer of PS and not the sn-2,3-glycerol stereoisomer. Both flippase and Atp8a1 activities are insensitive to the stereochemistry of the serine headgroup. Most modifications of the PS headgroup structure decrease recognition by the plasma membrane PS flippase. Activation of Atp8a1 is also reduced by these modifications; phosphatidylserine-O-methyl ester, lysophosphatidylserine, glycerophosphoserine, and phosphoserine, which are not transported by the plasma membrane flippase, do not activate Atp8a1. Weakly translocated lipids (PE, phosphatidylhydroxypropionate, and phosphatidylhomoserine) are also weak Atp8a1 activators. However, N-methyl-phosphatidylserine, which is transported by the plasma membrane flippase at a rate equivalent to PS, is incapable of activating Atp8a1 activity. These results indicate that the ATPase activity of the secretory granule Atp8a1 is activated by phospholipids binding to a specific site whose properties (PS selectivity, dependence upon glycerol but not serine, stereochemistry, and vanadate sensitivity) are similar to, but distinct from, the properties of the substrate binding site of the plasma membrane flippase.''' data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala ... val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentencer = new SentenceDetector() .setInputCols("document") .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val drugprot_ner_tagger = MedicalNerModel.pretrained("ner_drugprot_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val pos_tagger = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") // Set a filter on pairs of named entities which will be treated as relation candidates val drugprot_re_ner_chunk_filter = new RENerChunksFilter() .setInputCols(Array("ner_chunks", "dependencies")) .setMaxSyntacticDistance(10) .setOutputCol("re_ner_chunks") // .setRelationPairs(Array("CHEMICAL-GENE")) // This model can also be trained on document-level relations - in which case, while predicting, use "document" instead of "sentence" as input. val drugprot_re_Model = RelationExtractionDLModel() .pretrained("redl_drugprot_biobert", "en", "clinical/models") .setPredictionThreshold(0.9) .setInputCols(Array("re_ner_chunks", "sentences")) .setOutputCol("relations") val pipeline = new Pipeline().setStages(Array(documenter, sentencer, tokenizer, words_embedder, drugprot_ner_tagger, ner_converter, pos_tagger, dependency_parser, drugprot_re_ner_chunk_filter, drugprot_re_Model)) val data = Seq("""Lipid specific activation of the murine P4-ATPase Atp8a1 (ATPase II). The asymmetric transbilayer distribution of phosphatidylserine (PS) in the mammalian plasma membrane and secretory vesicles is maintained, in part, by an ATP-dependent transporter. This aminophospholipid "flippase" selectively transports PS to the cytosolic leaflet of the bilayer and is sensitive to vanadate, Ca(2+), and modification by sulfhydryl reagents. Although the flippase has not been positively identified, a subfamily of P-type ATPases has been proposed to function as transporters of amphipaths, including PS and other phospholipids. A candidate PS flippase ATP8A1 (ATPase II), originally isolated from bovine secretory vesicles, is a member of this subfamily based on sequence homology to the founding member of the subfamily, the yeast protein Drs2, which has been linked to ribosomal assembly, the formation of Golgi-coated vesicles, and the maintenance of PS asymmetry. To determine if ATP8A1 has biochemical characteristics consistent with a PS flippase, a murine homologue of this enzyme was expressed in insect cells and purified. The purified Atp8a1 is inactive in detergent micelles or in micelles containing phosphatidylcholine, phosphatidic acid, or phosphatidylinositol, is minimally activated by phosphatidylglycerol or phosphatidylethanolamine (PE), and is maximally activated by PS. The selectivity for PS is dependent upon multiple elements of the lipid structure. Similar to the plasma membrane PS transporter, Atp8a1 is activated only by the naturally occurring sn-1,2-glycerol isomer of PS and not the sn-2,3-glycerol stereoisomer. Both flippase and Atp8a1 activities are insensitive to the stereochemistry of the serine headgroup. Most modifications of the PS headgroup structure decrease recognition by the plasma membrane PS flippase. Activation of Atp8a1 is also reduced by these modifications; phosphatidylserine-O-methyl ester, lysophosphatidylserine, glycerophosphoserine, and phosphoserine, which are not transported by the plasma membrane flippase, do not activate Atp8a1. Weakly translocated lipids (PE, phosphatidylhydroxypropionate, and phosphatidylhomoserine) are also weak Atp8a1 activators. However, N-methyl-phosphatidylserine, which is transported by the plasma membrane flippase at a rate equivalent to PS, is incapable of activating Atp8a1 activity. These results indicate that the ATPase activity of the secretory granule Atp8a1 is activated by phospholipids binding to a specific site whose properties (PS selectivity, dependence upon glycerol but not serine, stereochemistry, and vanadate sensitivity) are similar to, but distinct from, the properties of the substrate binding site of the plasma membrane flippase.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.drugprot").predict("""Lipid specific activation of the murine P4-ATPase Atp8a1 (ATPase II). The asymmetric transbilayer distribution of phosphatidylserine (PS) in the mammalian plasma membrane and secretory vesicles is maintained, in part, by an ATP-dependent transporter. This aminophospholipid "flippase" selectively transports PS to the cytosolic leaflet of the bilayer and is sensitive to vanadate, Ca(2+), and modification by sulfhydryl reagents. Although the flippase has not been positively identified, a subfamily of P-type ATPases has been proposed to function as transporters of amphipaths, including PS and other phospholipids. A candidate PS flippase ATP8A1 (ATPase II), originally isolated from bovine secretory vesicles, is a member of this subfamily based on sequence homology to the founding member of the subfamily, the yeast protein Drs2, which has been linked to ribosomal assembly, the formation of Golgi-coated vesicles, and the maintenance of PS asymmetry. To determine if ATP8A1 has biochemical characteristics consistent with a PS flippase, a murine homologue of this enzyme was expressed in insect cells and purified. The purified Atp8a1 is inactive in detergent micelles or in micelles containing phosphatidylcholine, phosphatidic acid, or phosphatidylinositol, is minimally activated by phosphatidylglycerol or phosphatidylethanolamine (PE), and is maximally activated by PS. The selectivity for PS is dependent upon multiple elements of the lipid structure. Similar to the plasma membrane PS transporter, Atp8a1 is activated only by the naturally occurring sn-1,2-glycerol isomer of PS and not the sn-2,3-glycerol stereoisomer. Both flippase and Atp8a1 activities are insensitive to the stereochemistry of the serine headgroup. Most modifications of the PS headgroup structure decrease recognition by the plasma membrane PS flippase. Activation of Atp8a1 is also reduced by these modifications; phosphatidylserine-O-methyl ester, lysophosphatidylserine, glycerophosphoserine, and phosphoserine, which are not transported by the plasma membrane flippase, do not activate Atp8a1. Weakly translocated lipids (PE, phosphatidylhydroxypropionate, and phosphatidylhomoserine) are also weak Atp8a1 activators. However, N-methyl-phosphatidylserine, which is transported by the plasma membrane flippase at a rate equivalent to PS, is incapable of activating Atp8a1 activity. These results indicate that the ATPase activity of the secretory granule Atp8a1 is activated by phospholipids binding to a specific site whose properties (PS selectivity, dependence upon glycerol but not serine, stereochemistry, and vanadate sensitivity) are similar to, but distinct from, the properties of the substrate binding site of the plasma membrane flippase.""") ```

## Results ```bash +---------+--------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ | relation| entity1|entity1_begin|entity1_end| chunk1|entity2|entity2_begin|entity2_end| chunk2|confidence| +---------+--------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ |SUBSTRATE|CHEMICAL| 308| 310| PS| GENE| 275| 283| flippase| 0.998399| |ACTIVATOR|CHEMICAL| 1563| 1578| sn-1,2-glycerol| GENE| 1479| 1509|plasma membrane P...| 0.999304| |ACTIVATOR|CHEMICAL| 1563| 1578| sn-1,2-glycerol| GENE| 1511| 1517| Atp8a1| 0.979057| |ACTIVATOR|CHEMICAL| 2112| 2114| PE| GENE| 2189| 2195| Atp8a1| 0.998299| |ACTIVATOR|CHEMICAL| 2116| 2145|phosphatidylhydro...| GENE| 2189| 2195| Atp8a1| 0.981534| |ACTIVATOR|CHEMICAL| 2151| 2173|phosphatidylhomos...| GENE| 2189| 2195| Atp8a1| 0.988504| |SUBSTRATE|CHEMICAL| 2217| 2244|N-methyl-phosphat...| GENE| 2290| 2298| flippase| 0.994092| |ACTIVATOR|CHEMICAL| 1292| 1312|phosphatidylglycerol| GENE| 1134| 1140| Atp8a1| 0.994409| |ACTIVATOR|CHEMICAL| 1316| 1340|phosphatidylethan...| GENE| 1134| 1140| Atp8a1| 0.988359| |ACTIVATOR|CHEMICAL| 1342| 1344| PE| GENE| 1134| 1140| Atp8a1| 0.988399| |ACTIVATOR|CHEMICAL| 1377| 1379| PS| GENE| 1134| 1140| Atp8a1| 0.996349| |ACTIVATOR|CHEMICAL| 2526| 2528| PS| GENE| 2444| 2450| Atp8a1| 0.978597| |ACTIVATOR|CHEMICAL| 2526| 2528| PS| GENE| 2403| 2409| ATPase| 0.988679| +---------+--------+-------------+-----------+--------------------+-------+-------------+-----------+--------------------+----------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_drugprot_biobert| |Compatibility:|Healthcare NLP 3.3.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|405.4 MB| ## Data Source This model was trained on the [DrugProt corpus](https://zenodo.org/record/5119892). ## Benchmarking ```bash label recall precision f1 support ACTIVATOR 0.885 0.776 0.827 235 AGONIST 0.810 0.925 0.864 137 ANTAGONIST 0.970 0.919 0.944 199 DIRECT-REGULATOR 0.836 0.901 0.867 403 INDIRECT-DOWNREGULATOR 0.885 0.850 0.867 313 INDIRECT-UPREGULATOR 0.844 0.887 0.865 270 INHIBITOR 0.947 0.937 0.942 1083 PART-OF 0.939 0.889 0.913 247 PRODUCT-OF 0.697 0.953 0.805 145 SUBSTRATE 0.912 0.884 0.898 468 Avg 0.873 0.892 0.879 - Weighted-Avg 0.897 0.899 0.897 - ``` --- layout: model title: Legal Assertion Status (Negation) author: John Snow Labs name: legassertion_negation date: 2023-01-01 tags: [negation, en, licensed] task: Assertion Status language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Legal Negation model, aimed to identify if an NER entity is mentioned in the context to be negated or not. ## Predicted Entities `positive`, `negative` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legassertion_negation_en_1.0.0_3.0_1672578547085.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legassertion_negation_en_1.0.0_3.0_1672578547085.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python import pyspark.sql.functions as F document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = nlp.SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner = legal.NerModel.pretrained("legner_orgs_prods_alias","en","legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = nlp.NerConverter() \ .setInputCols(["sentence", "token", "ner"]) \ .setOutputCol("ner_chunk") legassertion = legal.AssertionDLModel.pretrained("legassertion_negation", "en", "legal/models")\ .setInputCols(["sentence", "ner_chunk", "embeddings"])\ .setOutputCol("leglabel") pipe = nlp.Pipeline(stages = [ document_assembler, sentence_detector, tokenizer, embeddings, ner, ner_converter, legassertion]) text = "Gradio INC will not be entering into a joint agreement with Hugging Face, Inc." sdf = spark.createDataFrame([[text]]).toDF("text") res = pipe.fit(sdf).transform(sdf) res.select(F.explode(F.arrays_zip(res.ner_chunk.result, res.leglabel.result)).alias("cols"))\ .select(F.expr("cols['0']").alias("ner_chunk"), F.expr("cols['1']").alias("assertion")).show(200, truncate=100) ```

## Results ```bash +-----------------+---------+ | ner_chunk|assertion| +-----------------+---------+ | Gradio INC| negative| |Hugging Face, Inc| positive| +-----------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legassertion_negation| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, chunk, embeddings]| |Output Labels:|[assertion]| |Language:|en| |Size:|2.2 MB| ## References In-house annotated legal sentences ## Benchmarking ```bash label tp fp fn prec rec f1 negative 26 0 1 1.0 0.962963 0.9811321 positive 38 1 0 0.974359 1.0 0.987013 Macro-average 641 1 1 0.9871795 0.9814815 0.9843222 Micro-average 0.9846154 0.9846154 0.9846154 ``` --- layout: model title: German asr_exp_w2v2t_r_wav2vec2_s37 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2t_r_wav2vec2_s37 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2t_r_wav2vec2_s37` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2t_r_wav2vec2_s37_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_r_wav2vec2_s37_de_4.2.0_3.0_1664107978613.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2t_r_wav2vec2_s37_de_4.2.0_3.0_1664107978613.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2t_r_wav2vec2_s37", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2t_r_wav2vec2_s37", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2t_r_wav2vec2_s37| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: Vietnamese Lemmatizer author: John Snow Labs name: lemma date: 2021-04-02 tags: [vi, open_source, lemmatizer] task: Lemmatization language: vi edition: Spark NLP 3.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a dictionary-based lemmatizer that assigns all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_vi_3.0.0_3.0_1617388850136.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_vi_3.0.0_3.0_1617388850136.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----+ |lemma| +-----+ | Tất| | cả| | đều| | hồi| | hộp| | .| +-----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|vi| ## Data Source The model was trained on the [Universal Dependencies](https://www.universaldependencies.org) version 2.7. ## Benchmarking ```bash Precision=0.96, Recall=0.89, F1-score=0.93 ``` --- layout: model title: Japanese Electra Embeddings (from izumi-lab) author: John Snow Labs name: electra_embeddings_electra_base_japanese_generator date: 2022-05-17 tags: [ja, open_source, electra, embeddings] task: Embeddings language: ja edition: Spark NLP 3.4.4 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Electra Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-japanese-generator` is a Japanese model orginally trained by `izumi-lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_japanese_generator_ja_3.4.4_3.0_1652786593799.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_embeddings_electra_base_japanese_generator_ja_3.4.4_3.0_1652786593799.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_embeddings_electra_base_japanese_generator| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|ja| |Size:|132.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/izumi-lab/electra-base-japanese-generator - https://github.com/google-research/electra - https://github.com/retarfi/language-pretraining/tree/v1.0 - https://github.com/google-research/electra - https://arxiv.org/abs/2003.10555 - https://creativecommons.org/licenses/by-sa/4.0/ --- layout: model title: Turkish BertForQuestionAnswering model (from husnu) author: John Snow Labs name: bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3 date: 2022-06-02 tags: [open_source, question_answering, bert] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-turkish-cased-finetuned_lr-2e-05_epochs-3` is a Turkish model orginally trained by `husnu`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3_tr_4.0.0_3.0_1654180690796.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3_tr_4.0.0_3.0_1654180690796.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3","tr") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.bert.base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_turkish_cased_finetuned_lr_2e_05_epochs_3| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|tr| |Size:|412.8 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/husnu/bert-base-turkish-cased-finetuned_lr-2e-05_epochs-3 --- layout: model title: Legal Securities Purchase Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_securities_purchase_agreement_bert date: 2022-11-25 tags: [en, legal, classification, agreement, securities_purchase, licensed, bert] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_securities_purchase_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `securities-purchase-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `securities-purchase-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_securities_purchase_agreement_bert_en_1.0.0_3.0_1669369230699.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_securities_purchase_agreement_bert_en_1.0.0_3.0_1669369230699.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_securities_purchase_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[securities-purchase-agreement]| |[other]| |[other]| |[securities-purchase-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_securities_purchase_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.6 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support other 0.93 0.95 0.94 65 securities-purchase-agreement 0.93 0.89 0.91 45 accuracy - - 0.93 110 macro-avg 0.93 0.92 0.92 110 weighted-avg 0.93 0.93 0.93 110 ``` --- layout: model title: English BertForQuestionAnswering model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-4` is a English model orginally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4_en_4.0.0_3.0_1654189557504.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4_en_4.0.0_3.0_1654189557504.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.span_bert.base_cased_1024d_seed_4").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_1024_finetuned_squad_seed_4| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|390.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-1024-finetuned-squad-seed-4 --- layout: model title: Intent Classification for Airline Traffic Information System queries (ATIS dataset) author: John Snow Labs name: classifierdl_use_atis date: 2021-01-25 task: Text Classification language: en nav_key: models edition: Spark NLP 2.7.1 spark_version: 2.4 tags: [en, classifier, open_source] supported: true annotator: ClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Classify user questions into 5 categories of an airline traffic information system. ## Predicted Entities `atis_abbreviation`, `atis_airfare`, `atis_airline`, `atis_flight`, `atis_ground_service` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/classifierdl_use_atis_en_2.7.1_2.4_1611572512585.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/classifierdl_use_atis_en_2.7.1_2.4_1611572512585.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------------------------------------------+----------------+ | document | class | +-------------------------------------------------------------------+----------------+ | what is the price of flight from newyork to washington | atis_airfare | | how many flights does twa have in business class | atis_quantity | +-------------------------------------------------------------------+----------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|classifierdl_use_atis| |Compatibility:|Spark NLP 2.7.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Dependencies:|tfhub_use| ## Data Source This model is trained on data obtained from https://www.kaggle.com/hassanamin/atis-airlinetravelinformationsystem ## Benchmarking ```bash precision recall f1-score support atis_abbreviation 1.00 1.00 1.00 33 atis_airfare 0.60 0.96 0.74 48 atis_airline 0.69 0.89 0.78 38 atis_flight 0.99 0.93 0.96 632 atis_ground_service 1.00 1.00 1.00 36 accuracy 0.93 787 macro avg 0.86 0.96 0.90 787 weighted avg 0.95 0.93 0.94 787 ``` --- layout: model title: English RobertaForQuestionAnswering (from anas-awadalla) author: John Snow Labs name: roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-few-shot-k-64-finetuned-squad-seed-8` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8_en_4.0.0_3.0_1655733637029.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8_en_4.0.0_3.0_1655733637029.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.base_64d_seed_8").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_base_few_shot_k_64_finetuned_squad_seed_8| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|419.4 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/roberta-base-few-shot-k-64-finetuned-squad-seed-8 --- layout: model title: Arabic Part of Speech Tagger (DA-Dialectal Arabic dataset, Gulf Arabic POS) author: John Snow Labs name: bert_pos_bert_base_arabic_camelbert_da_pos_glf date: 2022-04-26 tags: [bert, pos, part_of_speech, ar, open_source] task: Part of Speech Tagging language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Part of Speech model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-arabic-camelbert-da-pos-glf` is a Arabic model orginally trained by `CAMeL-Lab`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_glf_ar_3.4.2_3.0_1650993664822.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_pos_bert_base_arabic_camelbert_da_pos_glf_ar_3.4.2_3.0_1650993664822.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_pos_bert_base_arabic_camelbert_da_pos_glf| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|ar| |Size:|407.6 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-da-pos-glf - https://camel.abudhabi.nyu.edu/annotated-gumar-corpus/ - https://arxiv.org/abs/2103.06678 - https://github.com/CAMeL-Lab/CAMeLBERT - https://github.com/CAMeL-Lab/camel_tools --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_squadv2_base_3_epochs date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `squadv2-roberta-base-3-epochs` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_squadv2_base_3_epochs_en_4.3.0_3.0_1674224185355.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_squadv2_base_3_epochs_en_4.3.0_3.0_1674224185355.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squadv2_base_3_epochs","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_squadv2_base_3_epochs","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_squadv2_base_3_epochs| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|460.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/squadv2-roberta-base-3-epochs --- layout: model title: Pipeline to Detect Chemical Compounds and Genes (BertForTokenClassifier) author: John Snow Labs name: bert_token_classifier_ner_chemprot_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, berfortokenclassification, chemprot, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [bert_token_classifier_ner_chemprot](https://nlp.johnsnowlabs.com/2022/01/06/bert_token_classifier_ner_chemprot_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_3.0_1647889733985.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/bert_token_classifier_ner_chemprot_pipeline_en_3.4.1_3.0_1647889733985.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` ```scala val pipeline = new PretrainedPipeline("bert_token_classifier_ner_chemprot_pipeline", "en", "clinical/models") pipeline.annotate("Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.") ``` {:.nlu-block} ```python import nlu nlu.load("en.classify.token_bert.chemprot_pipeline").predict("""Keratinocyte growth factor and acidic fibroblast growth factor are mitogens for primary cultures of mammary epithelium.""") ```

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distil_bert_finetuned_log_parser_winlogbeat","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_distil_bert_finetuned_log_parser_winlogbeat","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.distil_bert.log_parser_winlogbeat.by_Slavka").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_distil_bert_finetuned_log_parser_winlogbeat| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Slavka/distil-bert-finetuned-log-parser-winlogbeat --- layout: model title: English asr_model_sid_voxforge_cetuc_1 TFWav2Vec2ForCTC from joaoalvarenga author: John Snow Labs name: pipeline_asr_model_sid_voxforge_cetuc_1 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_model_sid_voxforge_cetuc_1` is a English model originally trained by joaoalvarenga. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_model_sid_voxforge_cetuc_1_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_sid_voxforge_cetuc_1_en_4.2.0_3.0_1664021895154.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_model_sid_voxforge_cetuc_1_en_4.2.0_3.0_1664021895154.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_model_sid_voxforge_cetuc_1', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_model_sid_voxforge_cetuc_1", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_model_sid_voxforge_cetuc_1| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Legal Amendments Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_amendments_bert date: 2023-03-05 tags: [en, legal, classification, clauses, amendments, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Amendments` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Amendments`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_amendments_bert_en_1.0.0_3.0_1678049943783.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_amendments_bert_en_1.0.0_3.0_1678049943783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_amendments_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Amendments]| |[Other]| |[Other]| |[Amendments]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_amendments_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.7 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Amendments 0.96 0.94 0.95 181 Other 0.95 0.97 0.96 206 accuracy - - 0.95 387 macro-avg 0.95 0.95 0.95 387 weighted-avg 0.95 0.95 0.95 387 ``` --- layout: model title: Italian Bert Embeddings (Cased) author: John Snow Labs name: bert_embeddings_bert_base_italian_xxl_cased date: 2022-04-11 tags: [bert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true recommended: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-italian-xxl-cased` is a Italian model orginally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_italian_xxl_cased_it_3.4.2_3.0_1649676734473.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_italian_xxl_cased_it_3.4.2_3.0_1649676734473.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_italian_xxl_cased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|415.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-xxl-cased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: Named Entity Recognition Profiling (Biobert) author: John Snow Labs name: ner_profiling_biobert date: 2022-08-28 tags: [ner, ner_profiling, biobert, licensed, en] task: [Named Entity Recognition, Pipeline Healthcare] language: en nav_key: models edition: Healthcare NLP 4.0.2 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pipeline can be used to explore all the available pretrained NER models at once. When you run this pipeline over your text, you will end up with the predictions coming out of each pretrained clinical NER model trained with `biobert_pubmed_base_cased`. It has been updated by adding NER model outputs to the previous version. Here are the NER models that this pretrained pipeline includes: `jsl_ner_wip_greedy_biobert`, `jsl_rd_ner_wip_greedy_biobert`, `ner_ade_biobert`, `ner_anatomy_biobert`, `ner_anatomy_coarse_biobert`, `ner_bionlp_biobert`, `ner_cellular_biobert`, `ner_chemprot_biobert`, `ner_clinical_biobert`, `ner_deid_biobert`, `ner_deid_enriched_biobert`, `ner_diseases_biobert`, `ner_events_biobert`, `ner_human_phenotype_gene_biobert`, `ner_human_phenotype_go_biobert`, `ner_jsl_biobert`, `ner_jsl_enriched_biobert`, `ner_jsl_greedy_biobert`, `ner_living_species_biobert`, `ner_posology_biobert`, `ner_posology_large_biobert`, `ner_risk_factors_biobert` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/11.2.Pretrained_NER_Profiling_Pipelines.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_4.0.2_3.0_1661701458686.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_profiling_biobert_en_4.0.2_3.0_1661701458686.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline ner_profiling_pipeline = PretrainedPipeline('ner_profiling_biobert', 'en', 'clinical/models') result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val ner_profiling_pipeline = PretrainedPipeline("ner_profiling_biobert", "en", "clinical/models") val result = ner_profiling_pipeline.annotate("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.profiling_biobert").predict("""A 28-year-old female with a history of gestational diabetes mellitus diagnosed eight years prior to presentation and subsequent type two diabetes mellitus ( T2DM ), one prior episode of HTG-induced pancreatitis three years prior to presentation , associated with an acute hepatitis , and obesity with a body mass index ( BMI ) of 33.5 kg/m2 , presented with a one-week history of polyuria , polydipsia , poor appetite , and vomiting .""") ```

## Results ```bash ******************** ner_diseases_biobert Model Results ******************** [('gestational diabetes mellitus', 'Disease'), ('type two diabetes mellitus', 'Disease'), ('T2DM', 'Disease'), ('HTG-induced pancreatitis', 'Disease'), ('hepatitis', 'Disease'), ('obesity', 'Disease'), ('polyuria', 'Disease'), ('polydipsia', 'Disease'), ('poor appetite', 'Disease'), ('vomiting', 'Disease')] ******************** ner_events_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('eight years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('three years', 'DURATION'), ('presentation', 'OCCURRENCE'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index', 'TEST'), ('BMI', 'TEST'), ('presented', 'OCCURRENCE'), ('a one-week', 'DURATION'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_jsl_biobert Model Results ******************** [('28-year-old', 'Age'), ('female', 'Gender'), ('gestational diabetes mellitus', 'Diabetes'), ('eight years prior', 'RelativeDate'), ('type two diabetes mellitus', 'Diabetes'), ('T2DM', 'Disease_Syndrome_Disorder'), ('HTG-induced pancreatitis', 'Disease_Syndrome_Disorder'), ('three years prior', 'RelativeDate'), ('acute', 'Modifier'), ('hepatitis', 'Disease_Syndrome_Disorder'), ('obesity', 'Obesity'), ('body mass index', 'BMI'), ('BMI ) of 33.5 kg/m2', 'BMI'), ('one-week', 'Duration'), ('polyuria', 'Symptom'), ('polydipsia', 'Symptom'), ('poor appetite', 'Symptom'), ('vomiting', 'Symptom')] ******************** ner_clinical_biobert Model Results ******************** [('gestational diabetes mellitus', 'PROBLEM'), ('subsequent type two diabetes mellitus ( T2DM', 'PROBLEM'), ('HTG-induced pancreatitis', 'PROBLEM'), ('an acute hepatitis', 'PROBLEM'), ('obesity', 'PROBLEM'), ('a body mass index ( BMI )', 'TEST'), ('polyuria', 'PROBLEM'), ('polydipsia', 'PROBLEM'), ('poor appetite', 'PROBLEM'), ('vomiting', 'PROBLEM')] ******************** ner_risk_factors_biobert Model Results ******************** [('diabetes mellitus', 'DIABETES'), ('subsequent type two diabetes mellitus', 'DIABETES'), ('obesity', 'OBESE')] ... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_profiling_biobert| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.0.2+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|766.4 MB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - BertEmbeddings - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - MedicalNerModel - NerConverter - Finisher --- layout: model title: T5 Clinical Summarization / QA model author: John Snow Labs name: t5_base_pubmedqa date: 2022-10-25 tags: [t5, licensed, clinical, en] task: Summarization language: en nav_key: models edition: Spark NLP for Healthcare 4.1.0 spark_version: [3.0, 3.2] supported: true annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The T5 transformer model described in the seminal paper “Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer” can perform a variety of tasks, such as text summarization, question answering and translation. More details about using the model can be found in the paper (https://arxiv.org/pdf/1910.10683.pdf). This model is specifically trained on medical data for text summarization and question answering. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/t5_base_pubmedqa_en_4.1.0_3.0_1666670271455.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/t5_base_pubmedqa_en_4.1.0_3.0_1666670271455.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("documents") t5 = T5Transformer().pretrained("t5_base_pubmedqa", "en", "clinical/models")\ .setInputCols(["documents"])\ .setOutputCol("t5_output")\ .setTask("summarize medical questions:")\ .setMaxOutputLength(200) pipeline = Pipeline(stages=[document_assembler, t5]) data = spark.createDataFrame([ [1, "content:SUBJECT: Normal physical traits but no period MESSAGE: I'm a 40 yr. old woman that has infantile reproductive organs and have never experienced a mensus. I have had Doctors look but they all say I just have infantile female reproductive organs. When I try to look for answers on the internet I cannot find anything. ALL my \"girly\" parts are normal. My organs never matured. Could you give me more information please. focus:all"] ]).toDF('id', 'text') results = pipeline.fit(data).transform(data) results.select("t5_output.result").show(truncate=False) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("documents") val t5 = T5Transformer() .pretrained("t5_base_pubmedqa", "en", "clinical/models") .setInputCols("documents") .setOutputCol("t5_output") .setTask("summarize medical questions:") .setMaxOutputLength(200) val pipeline = new Pipeline().setStages(Array(document_assembler, t5)) val data = Seq(Array( (1, "content:SUBJECT: Normal physical traits but no period MESSAGE: I'm a 40 yr. old woman that has infantile reproductive organs and have never experienced a mensus. I have had Doctors look but they all say I just have infantile female reproductive organs. When I try to look for answers on the internet I cannot find anything. ALL my \"girly\" parts are normal. My organs never matured. Could you give me more information please. focus:all") )).toDF("id", "text") val results = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.t5.base_pubmedqa").predict("""content:SUBJECT: Normal physical traits but no period MESSAGE: I'm a 40 yr. old woman that has infantile reproductive organs and have never experienced a mensus. I have had Doctors look but they all say I just have infantile female reproductive organs. When I try to look for answers on the internet I cannot find anything. ALL my \""") ```

## Results ```bash I have a normal physical appearance and have no mensus. Can you give me more information? ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_base_pubmedqa| |Compatibility:|Spark NLP for Healthcare 4.1.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|916.7 MB| ## References Trained on Pubmed data & qnli --- layout: model title: Summarize clinical notes author: John Snow Labs name: summarizer_clinical_jsl date: 2023-03-25 tags: [en, licensed, clinical, summarization, tensorflow] task: Summarization language: en edition: Healthcare NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: MedicalSummarizer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a modified version of Flan-T5 (LLM) based summarization model that is finetuned with clinical notes, encounters, critical care notes, discharge notes, reports, curated by John Snow Labs. This model is further optimized by augmenting the training methodology, and dataset. It can generate summaries from clinical notes up to 512 tokens given the input text (max 1024 tokens). ## Predicted Entities {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/MEDICAL_TEXT_SUMMARIZATION/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/32.Medical_Text_Summarization.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_en_4.3.1_3.0_1679772340755.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/summarizer_clinical_jsl_en_4.3.1_3.0_1679772340755.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") summarizer = MedicalSummarizer()\ .pretrained("summarizer_clinical_jsl", "en", "clinical/models")\ .setInputCols("document")\ .setOutputCol("summary")\ .setMaxTextLength(512)\ .setMaxNewTokens(512) pipeline = Pipeline(stages=[document,summarizer]) text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" data = spark.createDataFrame([[text]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val summarizer = MedicalSummarizer() .pretrained("summarizer_clinical_jsl", "en", "clinical/models") .setInputCols("document") .setOutputCol("summary") .setMaxTextLength(512) .setMaxNewTokens(512) val pipeline = new Pipeline().setStages(Array(document_assembler, summarizer)) val text = """Patient with hypertension, syncope, and spinal stenosis - for recheck. (Medical Transcription Sample Report) SUBJECTIVE: The patient is a 78-year-old female who returns for recheck. She has hypertension. She denies difficulty with chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. PAST MEDICAL HISTORY / SURGERY / HOSPITALIZATIONS: Reviewed and unchanged from the dictation on 12/03/2003. MEDICATIONS: Atenolol 50 mg daily, Premarin 0.625 mg daily, calcium with vitamin D two to three pills daily, multivitamin daily, aspirin as needed, and TriViFlor 25 mg two pills daily. She also has Elocon cream 0.1% and Synalar cream 0.01% that she uses as needed for rash.""" val data = Seq(text).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash A 78-year-old female with hypertension, syncope, and spinal stenosis returns for recheck. She denies chest pain, palpations, orthopnea, nocturnal dyspnea, or edema. She is on multiple medications and has Elocon cream and Synalar cream for rash. ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|summarizer_clinical_jsl| |Compatibility:|Healthcare NLP 4.3.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|920.1 MB| ## Benchmarkings ### Benchmark on MtSamples Summarization Dataset : | model_name | model_size | rouge | bleu | bertscore_precision | bertscore_recall: | bertscore_f1 | |--|--|--|--|--|--|--| philschmid/flan-t5-base-samsum | 250M | 0.1919 | 0.1124 | 0.8409 | 0.8964 | 0.8678 | linydub/bart-large-samsum | 500M | 0.1586 | 0.0732 | 0.8747 | 0.8184 | 0.8456 | philschmid/bart-large-cnn-samsum | 500M | 0.2170 | 0.1299 | 0.8846 | 0.8436 | 0.8636 | transformersbook/pegasus-samsum | 500M | 0.1924 | 0.0965 | 0.8920 | 0.8149 | 0.8517 | summarizer_clinical_jsl | 250M | 0.4836 | 0.4188 | 0.9041 | 0.9374 | 0.9204 | summarizer_clinical_jsl_augmented | 250M | 0.5119 | 0.4545 | 0.9282 | 0.9526 | 0.9402 | ### Benchmark on MIMIC Summarization Dataset : | model_name | model_size | rouge | bleu | bertscore_precision | bertscore_recall: | bertscore_f1 | |--|--|--|--|--|--|--| philschmid/flan-t5-base-samsum | 250M | 0.1910 | 0.1037 | 0.8708 | 0.9056 | 0.8879 | linydub/bart-large-samsum | 500M | 0.1252 | 0.0382 | 0.8933 | 0.8440 | 0.8679 | philschmid/bart-large-cnn-samsum | 500M | 0.1795 | 0.0889 | 0.9172 | 0.8978 | 0.9074 | transformersbook/pegasus-samsum | 570M | 0.1425 | 0.0582 | 0.9171 | 0.8682 | 0.8920 | summarizer_clinical_jsl | 250M | 0.395 | 0.2962 | 0.895 | 0.9316 | 0.913 | summarizer_clinical_jsl_augmented | 250M | 0.3964 | 0.307 | 0.9109 | 0.9452 | 0.9227 | ![Benchmark Summary](https://github.com/JohnSnowLabs/jsl-private-projects/blob/llm_benchmarks/internal_projects/LLM_Experiments/jsl-summarization-benchmarks.png?raw=true) ## References Trained on in-house curated dataset --- layout: model title: English BertForQuestionAnswering model (from Kutay) author: John Snow Labs name: bert_qa_fine_tuned_squad_aip date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `fine_tuned_squad_aip` is a English model orginally trained by `Kutay`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_fine_tuned_squad_aip_en_4.0.0_3.0_1654187684708.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_fine_tuned_squad_aip_en_4.0.0_3.0_1654187684708.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_fine_tuned_squad_aip","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_fine_tuned_squad_aip","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.by_Kutay").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_fine_tuned_squad_aip| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|407.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/Kutay/fine_tuned_squad_aip --- layout: model title: Recognize Entities OntoNotes pipeline - ELECTRA Base author: John Snow Labs name: onto_recognize_entities_electra_base date: 2021-03-23 tags: [open_source, english, onto_recognize_entities_electra_base, pipeline, en] supported: true task: [Named Entity Recognition, Lemmatization] language: en nav_key: models edition: Spark NLP 3.0.0 spark_version: 3.0 annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The onto_recognize_entities_electra_base is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entities . It performs most of the common text processing tasks on your dataframe {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/NER_EN_18/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_base_en_3.0.0_3.0_1616477573783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/onto_recognize_entities_electra_base_en_3.0.0_3.0_1616477573783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipelinein pipeline = PretrainedPipeline('onto_recognize_entities_electra_base', lang = 'en') annotations = pipeline.fullAnnotate(""Hello from John Snow Labs ! "")[0] annotations.keys() ``` ```scala val pipeline = new PretrainedPipeline("onto_recognize_entities_electra_base", lang = "en") val result = pipeline.fullAnnotate("Hello from John Snow Labs ! ")(0) ``` {:.nlu-block} ```python import nlu text = [""Hello from John Snow Labs ! ""] result_df = nlu.load('en.ner.onto.electra.base').predict(text) result_df ```

## Results ```bash | | document | sentence | token | embeddings | ner | entities | |---:|:---------------------------------|:--------------------------------|:-----------------------------------------------|:-----------------------------|:-------------------------------------------|:-------------------| | 0 | ['Hello from John Snow Labs ! '] | ['Hello from John Snow Labs !'] | ['Hello', 'from', 'John', 'Snow', 'Labs', '!'] | [[0.2088415920734405,.,...]] | ['O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'O'] | ['John Snow Labs'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|onto_recognize_entities_electra_base| |Type:|pipeline| |Compatibility:|Spark NLP 3.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| --- layout: model title: English BertForQuestionAnswering Cased model (from KFlash) author: John Snow Labs name: bert_qa_kflash_finetuned_squad_accelera date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-finetuned-squad-accelerate` is a English model originally trained by `KFlash`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_kflash_finetuned_squad_accelera_en_4.0.0_3.0_1657186938840.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_kflash_finetuned_squad_accelera_en_4.0.0_3.0_1657186938840.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_kflash_finetuned_squad_accelera","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_kflash_finetuned_squad_accelera","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_kflash_finetuned_squad_accelera| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/KFlash/bert-finetuned-squad-accelerate --- layout: model title: Chinese BertForMaskedLM Cased model (from uer) author: John Snow Labs name: bert_embeddings_chinese_roberta_l_6_h_768 date: 2022-12-06 tags: [zh, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: zh edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `chinese_roberta_L-6_H-768` is a Chinese model originally trained by `uer`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_768_zh_4.2.4_3.0_1670326011437.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_chinese_roberta_l_6_h_768_zh_4.2.4_3.0_1670326011437.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_768","zh") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_chinese_roberta_l_6_h_768","zh") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_chinese_roberta_l_6_h_768| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|zh| |Size:|224.1 MB| |Case sensitive:|true| ## References - https://huggingface.co/uer/chinese_roberta_L-6_H-768 - https://github.com/dbiir/UER-py/ - https://arxiv.org/abs/1909.05658 - https://arxiv.org/abs/1908.08962 - https://github.com/dbiir/UER-py/wiki/Modelzoo - https://github.com/CLUEbenchmark/CLUECorpus2020/ - https://github.com/dbiir/UER-py/ - https://cloud.tencent.com/ --- layout: model title: Detect Symptoms, Treatments and Other Entities in German author: John Snow Labs name: ner_healthcare date: 2020-09-28 task: Named Entity Recognition language: de edition: Healthcare NLP 2.6.0 spark_version: 2.4 tags: [ner, clinical, de, licensed] supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect symptoms, treatments and other entities in medical text in German language. ## Predicted Entities `DIAGLAB_PROCEDURE`, `MEDICAL_SPECIFICATION`, `MEDICAL_DEVICE`, `MEASUREMENT`, `BIOLOGICAL_CHEMISTRY`, `BODY_FLUID`, `TIME_INFORMATION`, `LOCAL_SPECIFICATION`, `BIOLOGICAL_PARAMETER`, `PROCESS`, `MEDICATION`, `DOSING`, `DEGREE`, `MEDICAL_CONDITION`, `PERSON`, `TISSUE`, `STATE_OF_HEALTH`, `BODY_PART`, `TREATMENT` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HEALTHCARE_DE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/NER_HEALTHCARE_DE.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}[Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_de_2.5.5_2.4_1599433028253.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_healthcare_de_2.5.5_2.4_1599433028253.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use Use as part of an nlp pipeline with the following stages: DocumentAssembler, SentenceDetector, Tokenizer, WordEmbeddingsModel, NerDLModel. Add the NerConverter to the end of the pipeline to convert entity tokens into full entity chunks.

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","de","clinical/models")\ .setInputCols(["document","token"])\ .setOutputCol("embeddings") clinical_ner = NerDLModel.pretrained("ner_healthcare", "de", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ... nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, clinical_ner_converter]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) annotations = light_pipeline.fullAnnotate("Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist ein hochmalignes bronchogenes Karzinom") ``` ```scala ... val word_embeddings = WordEmbeddingsModel.pretrained("w2v_cc_300d","de","clinical/models") .setInputCols(Array("document","token")) .setOutputCol("embeddings") val ner = NerDLModel.pretrained("ner_healthcare", "de", "clinical/models") .setInputCols("sentence", "token", "embeddings") .setOutputCol("ner") ... val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, clinical_ner_converter)) val data = Seq("Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist ein hochmalignes bronchogenes Karzinom").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("de.med_ner.healthcare").predict("""Das Kleinzellige Bronchialkarzinom (Kleinzelliger Lungenkrebs, SCLC) ist ein hochmalignes bronchogenes Karzinom""") ```

{:.h2_title} ## Results ```bash +----+-------------------+---------+---------+--------------------------+ | | chunk | begin | end | entity | +====+===================+=========+=========+==========================+ | 0 | Kleinzellige | 4 | 15 | MEDICAL_SPECIFICATION | +----+-------------------+---------+---------+--------------------------+ | 1 | Bronchialkarzinom | 17 | 33 | MEDICAL_CONDITION | +----+-------------------+---------+---------+--------------------------+ | 2 | Kleinzelliger | 36 | 48 | MEDICAL_SPECIFICATION | +----+-------------------+---------+---------+--------------------------+ | 3 | Lungenkrebs | 50 | 60 | MEDICAL_CONDITION | +----+-------------------+---------+---------+--------------------------+ | 4 | SCLC | 63 | 66 | MEDICAL_CONDITION | +----+-------------------+---------+---------+--------------------------+ | 5 | hochmalignes | 77 | 88 | MEASUREMENT | +----+-------------------+---------+---------+--------------------------+ | 6 | bronchogenes | 90 | 101 | BODY_PART | +----+-------------------+---------+---------+--------------------------+ | 7 | Karzinom | 103 | 110 | MEDICAL_CONDITION | +----+-------------------+---------+---------+--------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_healthcare| |Type:|ner| |Compatibility:|Healthcare NLP 2.6.0 +| |Edition:|Official| |License:|Licensed| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|[de]| |Case sensitive:|false| | Dependencies: | w2v_cc_300d | {:.h2_title} ## Data Source Trained on 2010 i2b2/VA challenge on concepts, assertions, and relations in clinical text with *w2v_cc_300d*. {:.h2_title} ## Benchmarking ```bash | | label | tp | fp | fn | precision| recall| f1 | |---:|--------------------:|-------:|------:|-----:|---------:|---------:|---------:| | 0 | BIOLOGICAL_PARAMETER| 103 | 52 | 57 | 0.6645 | 0.6438 | 0.654 | | 1 | BODY_FLUID | 166 | 16 | 24 | 0.9121 | 0.8737 | 0.8925 | | 2 | PERSON | 475 | 74 | 142 | 0.8652 | 0.7699 | 0.8148 | | 3 | DOSING | 38 | 14 | 31 | 0.7308 | 0.5507 | 0.6281 | | 4 | DIAGLAB_PROCEDURE | 236 | 58 | 68 | 0.8027 | 0.7763 | 0.7893 | | 5 | BODY_PART | 690 | 72 | 79 | 0.9055 | 0.8973 | 0.9014 | | 6 | MEDICATION | 391 | 117 | 167 | 0.7697 | 0.7007 | 0.7336 | | 7 | STATE_OF_HEALTH | 321 | 41 | 76 | 0.8867 | 0.8086 | 0.8458 | | 8 | LOCAL_SPECIFICATION | 57 | 19 | 24 | 0.75 | 0.7037 | 0.7261 | | 9 | MEASUREMENT | 574 | 260 | 222 | 0.6882 | 0.7211 | 0.7043 | | 10 | TREATMENT | 476 | 131 | 135 | 0.7842 | 0.7791 | 0.7816 | | 11 | MEDICAL_CONDITION | 1741 | 442 | 271 | 0.7975 | 0.8653 | 0.83 | | 12 | TIME_INFORMATION | 651 | 126 | 161 | 0.8378 | 0.8017 | 0.8194 | | 13 | BIOLOGICAL_CHEMISTRY| 192 | 55 | 60 | 0.7773 | 0.7619 | 0.7695 | ``` --- layout: model title: Zero-shot Legal NER (CUAD, large) author: John Snow Labs name: legner_roberta_zeroshot_cuad_large date: 2023-01-30 tags: [en, licensed, tensorflow] task: Named Entity Recognition language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true recommended: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a Zero-shot NER model, trained using Roberta on SQUAD and finetuned to perform Zero-shot NER using CUAD legal dataset. In order to use it, a specific prompt is required. This is an example of it for extracting PARTIES: ``` "Highlight the parts (if any) of this contract related to "Parties" that should be reviewed by a lawyer. Details: The two or more parties who signed the contract" ``` ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_roberta_zeroshot_cuad_large_en_1.0.0_3.0_1675089504749.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_roberta_zeroshot_cuad_large_en_1.0.0_3.0_1675089504749.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-----------------------+---------+ |chunk |ner_label| +-----------------------+---------+ |P.H. GLATFELTER COMPANY|PARTIES | +-----------------------+---------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_roberta_zeroshot_cuad_large| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|454.4 MB| |Case sensitive:|true| |Max sentence length:|128| ## References SQUAD and CUAD --- layout: model title: English DistilBertForTokenClassification Cased model (from Lucifermorningstar011) author: John Snow Labs name: distilbert_token_classifier_autotrain_ner_778023879 date: 2023-03-06 tags: [en, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: en edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `autotrain-ner-778023879` is a English model originally trained by `Lucifermorningstar011`. ## Predicted Entities `9`, `0` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1678134257673.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_autotrain_ner_778023879_en_4.3.1_3.0_1678134257673.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_autotrain_ner_778023879","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_autotrain_ner_778023879| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|244.1 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/Lucifermorningstar011/autotrain-ner-778023879 --- layout: model title: Spanish RoBERTa Embeddings (Base, Random Sampling, Using Sequence Length 512) author: John Snow Labs name: roberta_embeddings_bertin_base_random_exp_512seqlen date: 2022-04-14 tags: [roberta, embeddings, es, open_source] task: Embeddings language: es edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RoBERTa Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bertin-base-random-exp-512seqlen` is a Spanish model orginally trained by `bertin-project`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_random_exp_512seqlen_es_3.4.2_3.0_1649946088404.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_embeddings_bertin_base_random_exp_512seqlen_es_3.4.2_3.0_1649946088404.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_embeddings_bertin_base_random_exp_512seqlen| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|es| |Size:|230.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/bertin-project/bertin-base-random-exp-512seqlen --- layout: model title: English asr_wav2vec2_base_timit_demo_colab7_by_hassnain TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: asr_wav2vec2_base_timit_demo_colab7_by_hassnain date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab7_by_hassnain` is a English model originally trained by hassnain. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_base_timit_demo_colab7_by_hassnain_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab7_by_hassnain_en_4.2.0_3.0_1664018865478.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_base_timit_demo_colab7_by_hassnain_en_4.2.0_3.0_1664018865478.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_base_timit_demo_colab7_by_hassnain", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_base_timit_demo_colab7_by_hassnain", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_base_timit_demo_colab7_by_hassnain| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|355.0 MB| --- layout: model title: German XLMRobertaForTokenClassification Base Cased model (from lijingxin) author: John Snow Labs name: xlmroberta_ner_lijingxin_base_finetuned_panx date: 2022-08-13 tags: [de, open_source, xlm_roberta, ner] task: Named Entity Recognition language: de edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `xlm-roberta-base-finetuned-panx-de` is a German model originally trained by `lijingxin`. ## Predicted Entities `PER`, `LOC`, `ORG` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_lijingxin_base_finetuned_panx_de_4.1.0_3.0_1660435174962.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_lijingxin_base_finetuned_panx_de_4.1.0_3.0_1660435174962.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_lijingxin_base_finetuned_panx","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_lijingxin_base_finetuned_panx","de") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_lijingxin_base_finetuned_panx| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|de| |Size:|854.5 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/lijingxin/xlm-roberta-base-finetuned-panx-de - https://paperswithcode.com/sota?task=Token+Classification&dataset=xtreme --- layout: model title: English BertForQuestionAnswering model (from batterydata) author: John Snow Labs name: bert_qa_batteryonlybert_cased_squad_v1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `batteryonlybert-cased-squad-v1` is a English model orginally trained by `batterydata`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_batteryonlybert_cased_squad_v1_en_4.0.0_3.0_1654179335684.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_batteryonlybert_cased_squad_v1_en_4.0.0_3.0_1654179335684.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_batteryonlybert_cased_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_batteryonlybert_cased_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad_battery.bert.cased_only_bert.by_batterydata").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_batteryonlybert_cased_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|404.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/batterydata/batteryonlybert-cased-squad-v1 - https://github.com/ShuHuang/batterybert --- layout: model title: Recognize Entities Bert Noncontrib author: John Snow Labs name: recognize_entities_bert_noncontrib date: 2022-06-20 tags: [en, open_source] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The recognize_entities_bert_noncontrib is a pretrained pipeline that we can use to process text with a simple pipeline that performs basic processing steps and recognizes entites. It performs most of the common text processing tasks on your dataframe {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/recognize_entities_bert_noncontrib_en_4.0.0_3.0_1655697462434.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/recognize_entities_bert_noncontrib_en_4.0.0_3.0_1655697462434.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("recognize_entities_bert_noncontrib", "en", "clinical/models") result = pipeline.annotate("""I love johnsnowlabs! """) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|recognize_entities_bert_noncontrib| |Type:|pipeline| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|425.6 MB| ## Included Models - DocumentAssembler - SentenceDetector - TokenizerModel - BertEmbeddings - NerDLModel --- layout: model title: Legal Marketing Document Classifier (EURLEX) author: John Snow Labs name: legclf_marketing_bert date: 2023-03-06 tags: [en, legal, classification, clauses, marketing, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description European Union (EU) legislation is published in the EUR-Lex portal. All EU laws are annotated by the EU's Publications Office with multiple concepts from the EuroVoc thesaurus, a multilingual thesaurus maintained by the Publications Office. Given a document, the legclf_marketing_bert model, it is a Bert Sentence Embeddings Document Classifier, classifies if the document belongs to the class Marketing or not (Binary Classification) according to EuroVoc labels. ## Predicted Entities `Marketing`, `Other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_marketing_bert_en_1.0.0_3.0_1678111761414.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_marketing_bert_en_1.0.0_3.0_1678111761414.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_marketing_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Marketing]| |[Other]| |[Other]| |[Marketing]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_marketing_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.1 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Marketing 0.85 0.84 0.84 716 Other 0.82 0.83 0.83 648 accuracy - - 0.84 1364 macro-avg 0.84 0.84 0.84 1364 weighted-avg 0.84 0.84 0.84 1364 ``` --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from liuhaor4) author: John Snow Labs name: distilbert_qa_liuhaor4_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `liuhaor4`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_liuhaor4_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772009401.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_liuhaor4_base_uncased_finetuned_squad_en_4.3.0_3.0_1672772009401.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_liuhaor4_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_liuhaor4_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_liuhaor4_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/liuhaor4/distilbert-base-uncased-finetuned-squad --- layout: model title: Swedish BertForMaskedLM Cased model (from Addedk) author: John Snow Labs name: bert_embeddings_kb_distilled_cased date: 2022-12-02 tags: [sv, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: sv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `kbbert-distilled-cased` is a Swedish model originally trained by `Addedk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_kb_distilled_cased_sv_4.2.4_3.0_1670022547171.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_kb_distilled_cased_sv_4.2.4_3.0_1670022547171.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kb_distilled_cased","sv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_kb_distilled_cased","sv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_kb_distilled_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|sv| |Size:|308.3 MB| |Case sensitive:|true| ## References - https://huggingface.co/Addedk/kbbert-distilled-cased - https://spraakbanken.gu.se/en/resources/gigaword - https://github.com/AddedK/swedish-mbert-distillation/blob/main/azureML/pretrain_distillation.py - https://kth.diva-portal.org/smash/record.jsf?aq2=%5B%5B%5D%5D&c=2&af=%5B%5D&searchType=UNDERGRADUATE&sortOrder2=title_sort_asc&language=en&pid=diva2%3A1698451&aq=%5B%5B%7B%22freeText%22%3A%22added+kina%22%7D%5D%5D&sf=all&aqe=%5B%5D&sortOrder=author_sort_asc&onlyFullText=false&noOfRows=50&dswid=-6142 - https://arxiv.org/abs/2103.06418 - https://spraakbanken.gu.se/en/resources/gigaword --- layout: model title: English BertForQuestionAnswering model (from mrm8488) author: John Snow Labs name: bert_qa_bert_medium_wrslb_finetuned_squadv1 date: 2022-06-02 tags: [en, open_source, question_answering, bert] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-medium-wrslb-finetuned-squadv1` is a English model orginally trained by `mrm8488`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_wrslb_finetuned_squadv1_en_4.0.0_3.0_1654183753800.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_medium_wrslb_finetuned_squadv1_en_4.0.0_3.0_1654183753800.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_medium_wrslb_finetuned_squadv1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_medium_wrslb_finetuned_squadv1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.bert.medium").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_medium_wrslb_finetuned_squadv1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|154.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/mrm8488/bert-medium-wrslb-finetuned-squadv1 --- layout: model title: Turkish BertForQuestionAnswering Uncased model (from enelpi) author: John Snow Labs name: bert_qa_question_answering_uncased_squadv2 date: 2022-07-07 tags: [tr, open_source, bert, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-question-answering-uncased-squadv2_tr` is a Turkish model originally trained by `enelpi`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_uncased_squadv2_tr_4.0.0_3.0_1657187939073.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_question_answering_uncased_squadv2_tr_4.0.0_3.0_1657187939073.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_question_answering_uncased_squadv2","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_question_answering_uncased_squadv2","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_question_answering_uncased_squadv2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|413.2 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/enelpi/bert-question-answering-uncased-squadv2_tr --- layout: model title: Finnish asr_wav2vec2_large_xlsr_53_finnish_by_aapot TFWav2Vec2ForCTC from aapot author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_finnish_by_aapot date: 2022-09-24 tags: [wav2vec2, fi, audio, open_source, asr] task: Automatic Speech Recognition language: fi edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_finnish_by_aapot` is a Finnish model originally trained by aapot. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_finnish_by_aapot_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_aapot_fi_4.2.0_3.0_1664019226592.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_finnish_by_aapot_fi_4.2.0_3.0_1664019226592.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_aapot", "fi")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_finnish_by_aapot", "fi") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_finnish_by_aapot| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|fi| |Size:|1.2 GB| --- layout: model title: Stop Words Cleaner for Hindi author: John Snow Labs name: stopwords_hi date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: hi edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, hi] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_hi_hi_2.5.4_2.4_1594742439035.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_hi_hi_2.5.4_2.4_1594742439035.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_hi", "hi") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_hi", "hi") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""उत्तर के राजा होने के अलावा, जॉन स्नो एक अंग्रेजी चिकित्सक और संज्ञाहरण और चिकित्सा स्वच्छता के विकास में अग्रणी है।"""] stopword_df = nlu.load('hi.stopwords').predict(text) stopword_df[['cleanTokens']] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='उत्तर', metadata={'sentence': '0'}), Row(annotatorType='token', begin=9, end=12, result='राजा', metadata={'sentence': '0'}), Row(annotatorType='token', begin=22, end=26, result='अलावा', metadata={'sentence': '0'}), Row(annotatorType='token', begin=27, end=27, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=29, end=31, result='जॉन', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_hi| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|hi| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Legal Notices Clause Binary Classifier author: John Snow Labs name: legclf_notices_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `notices` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `notices` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_notices_clause_en_1.0.0_3.2_1660122758974.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_notices_clause_en_1.0.0_3.2_1660122758974.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_notices_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[notices]| |[other]| |[other]| |[notices]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_notices_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.3 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support notices 0.96 0.97 0.97 191 other 0.99 0.99 0.99 529 accuracy - - 0.98 720 macro-avg 0.98 0.98 0.98 720 weighted-avg 0.98 0.98 0.98 720 ``` --- layout: model title: Detect Genes and Human Phenotypes (biobert) author: John Snow Labs name: ner_human_phenotype_gene_biobert date: 2021-04-01 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect normalized mentions of genes (gene) and human phenotypes (hp) in medical text. ## Predicted Entities `HP`, `GENE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_HUMAN_PHENOTYPE_GENE_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_en_3.0.0_3.0_1617260636569.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_human_phenotype_gene_biobert_en_3.0.0_3.0_1617260636569.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_human_phenotype_gene_biobert", "en", "clinical/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings_clinical, clinical_ner, ner_converter]) model = nlpPipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["EXAMPLE_TEXT"]]).toDF("text")) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val embeddings_clinical = BertEmbeddings.pretrained("biobert_pubmed_base_cased") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_human_phenotype_gene_biobert", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings_clinical, ner, ner_converter)) val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.human_phenotype.gene_biobert").predict("""Put your text here.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_human_phenotype_gene_biobert| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|en| ## Benchmarking ```bash entity tp fp fn total precision recall f1 HP 1761.0 198.0 342.0 2103.0 0.8989 0.8374 0.8671 GENE 1600.0 290.0 361.0 1961.0 0.8466 0.8159 0.831 macro - - - - - - 0.8490 micro - - - - - - 0.8496 ``` --- layout: model title: Translate Esperanto to English Pipeline author: John Snow Labs name: translate_eo_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, eo, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `eo` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_eo_en_xx_2.7.0_2.4_1609687626260.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_eo_en_xx_2.7.0_2.4_1609687626260.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_eo_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_eo_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.eo.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_eo_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Bangla DistilBertForQuestionAnswering Cased model (from khasrul-alam) author: John Snow Labs name: distilbert_qa_banglabert_finetuned_squad date: 2023-01-03 tags: [bn, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: bn edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `banglabert-finetuned-squad` is a Bangla model originally trained by `khasrul-alam`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_banglabert_finetuned_squad_bn_4.3.0_3.0_1672765708608.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_banglabert_finetuned_squad_bn_4.3.0_3.0_1672765708608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_banglabert_finetuned_squad","bn")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_banglabert_finetuned_squad","bn") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_banglabert_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|bn| |Size:|247.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/khasrul-alam/banglabert-finetuned-squad --- layout: model title: English ElectraForQuestionAnswering model (from SauravMaheshkar) author: John Snow Labs name: electra_qa_base_chaii date: 2022-06-22 tags: [en, open_source, electra, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-base-chaii` is a English model originally trained by `SauravMaheshkar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_base_chaii_en_4.0.0_3.0_1655920382993.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_base_chaii_en_4.0.0_3.0_1655920382993.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_base_chaii","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_base_chaii","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.chaii.electra.base").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_base_chaii| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|408.5 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/SauravMaheshkar/electra-base-chaii --- layout: model title: English asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97 TFWav2Vec2ForCTC from chaitanya97 author: John Snow Labs name: asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97` is a English model originally trained by chaitanya97. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97_en_4.2.0_3.0_1664036036447.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97_en_4.2.0_3.0_1664036036447.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xls_r_300m_turkish_colab_by_chaitanya97| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Lemmatizer (Tagalog, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, tl] task: Lemmatization language: tl edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Tagalog Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_tl_3.4.1_3.0_1646316594942.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_tl_3.4.1_3.0_1646316594942.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------------------------------------------+ |result | +------------------------------------------+ |[Hindi, ka, mas, mahusay, kaysa, sa, akin]| +------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|tl| |Size:|3.2 KB| --- layout: model title: German asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756 date: 2022-09-26 tags: [wav2vec2, de, audio, open_source, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756` is a German model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756_de_4.2.0_3.0_1664190015002.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756_de_4.2.0_3.0_1664190015002.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756", "de")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756", "de") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_exp_w2v2r_vp_100k_accent_germany_0_austria_10_s756| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|de| |Size:|1.2 GB| --- layout: model title: ICD10 to UMLS Code Mapping author: John Snow Labs name: icd10cm_umls_mapping date: 2023-06-13 tags: [en, licensed, icd10cm, umls, pipeline, chunk_mapping] task: Chunk Mapping language: en edition: Healthcare NLP 4.4.4 spark_version: 3.2 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline maps ICD10CM codes to UMLS codes without using any text data. You’ll just feed white space-delimited ICD10CM codes and it will return the corresponding UMLS codes as a list. If there is no mapping, the original code is returned with no mapping. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_4.4.4_3.2_1686663524227.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/icd10cm_umls_mapping_en_4.4.4_3.2_1686663524227.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("icd10cm_umls_mapping", "en", "clinical/models") result = pipeline.fullAnnotate(['M8950', 'R822', 'R0901']) ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("icd10cm_umls_mapping", "en", "clinical/models") val result = pipeline.fullAnnotate(['M8950', 'R822', 'R0901']) ``` {:.nlu-block} ```python import nlu nlu.load("en.icd10cm.umls.mapping").predict("""Put your text here.""") ```

## Results ```bash Results {'icd10cm': ['M89.50', 'R82.2', 'R09.01'], 'umls': ['C4721411', 'C0159076', 'C0004044']} {:.model-param} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|icd10cm_umls_mapping| |Type:|pipeline| |Compatibility:|Healthcare NLP 4.4.4+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|956.6 KB| ## Included Models - DocumentAssembler - TokenizerModel - ChunkMapperModel --- layout: model title: English asr_maialong_model TFWav2Vec2ForCTC from buidung2004 author: John Snow Labs name: pipeline_asr_maialong_model date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_maialong_model` is a English model originally trained by buidung2004. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_maialong_model_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_maialong_model_en_4.2.0_3.0_1664023438768.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_maialong_model_en_4.2.0_3.0_1664023438768.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_maialong_model', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_maialong_model", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_maialong_model| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|227.7 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Bhojpuri Lemmatizer author: John Snow Labs name: lemma date: 2021-01-18 task: Lemmatization language: bh edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [bh, bho, open_source, lemmatizer] supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model uses context and language knowledge to assign all forms and inflections of a word to a single root. This enables the pipeline to treat the past and present tense of a verb, for example, as the same word instead of two completely different words. The lemmatizer takes into consideration the context surrounding a word to determine which root is correct when the word form alone is ambiguous. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/public/TEXT_PREPROCESSING/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/TEXT_PREPROCESSING.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_bh_2.7.0_2.4_1610989221391.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_bh_2.7.0_2.4_1610989221391.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash {'lemma': [Annotation(token, 0, 1, एह, {'sentence': '0'}), Annotation(token, 3, 7, आयोजन, {'sentence': '0'}), Annotation(token, 9, 11, में, {'sentence': '0'}), Annotation(token, 13, 17, विश्व, {'sentence': '0'}), Annotation(token, 19, 25, भोजपुरी, {'sentence': '0'}), Annotation(token, 27, 33, सम्मेलन, {'sentence': '0'}), Annotation(token, 35, 35, COMMA, {'sentence': '0'}), Annotation(token, 37, 45, पूर्वांचल, {'sentence': '0'}), Annotation(token, 47, 50, एकता, {'sentence': '0'}), Annotation(token, 52, 54, मंच, {'sentence': '0'}), Annotation(token, 56, 56, COMMA, {'sentence': '0'}), Annotation(token, 58, 60, वीर, {'sentence': '0'}), Annotation(token, 62, 66, कुँवर, {'sentence': '0'}), Annotation(token, 68, 71, सिंह, {'sentence': '0'}), Annotation(token, 73, 81, फाउन्डेशन, {'sentence': '0'}), Annotation(token, 83, 83, COMMA, {'sentence': '0'}), Annotation(token, 85, 93, पूर्वांचल, {'sentence': '0'}), Annotation(token, 95, 101, भोजपुरी, {'sentence': '0'}), Annotation(token, 103, 108, महासभा, {'sentence': '0'}), Annotation(token, 110, 110, COMMA, {'sentence': '0'}), Annotation(token, 112, 114, अउर, {'sentence': '0'}), Annotation(token, 116, 119, हर्फ, {'sentence': '0'}), Annotation(token, 121, 121, -, {'sentence': '0'}), Annotation(token, 123, 128, मीडिया, {'sentence': '0'}), Annotation(token, 130, 131, को, {'sentence': '0'}), Annotation(token, 133, 140, सहभागिता, {'sentence': '0'}), Annotation(token, 142, 143, बा, {'sentence': '0'}), Annotation(token, 145, 145, ।, {'sentence': '0'})]} ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[document]| |Output Labels:|[token]| |Language:|bh| ## Data Source The model was trained on the [Universal Dependencies](http://universaldependencies.org) data set version 2.7. Reference: - Ojha, A. K., & Zeman, D. (2020). Universal Dependency Treebanks for Low-Resource Indian Languages: The Case of Bhojpuri. Proceedings of the WILDRE5{--} 5th Workshop on Indian Language Data: Resources and Evaluation. --- layout: model title: Whereas Pipeline author: John Snow Labs name: legpipe_whereas date: 2022-08-24 tags: [en, legal, whereas, licensed] task: [Named Entity Recognition, Part of Speech Tagging, Dependency Parser, Relation Extraction] language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: Pipeline article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description IMPORTANT: Don't run this model on the whole legal agreement. Instead: - Split by paragraphs. You can use [notebook 1](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/tutorials/Certification_Trainings) in Finance or Legal as inspiration; - Use the `legclf_cuad_whereas_clause` Text Classifier to select only these paragraphs; This is a Pretrained Pipeline to show extraction of whereas clauses (Subject, Action and Object), and also the relationships between them, using two approaches: - A Semantic Relation Extraction Model; - A Dependency Parser Tree; The difficulty of these entities is that they are totally free-text, with OBJECT being sometimes very long with very diverse vocabulary. Although the NER and the REDL can help you identify them, the Dependency Parser has been added so you can navigate the tree looking for specific direct objects or other phrases. ## Predicted Entities `WHEREAS_SUBJECT`, `WHEREAS_OBJECT`, `WHEREAS_ACTION` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/LEGALNER_WHEREAS){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legpipe_whereas_en_1.0.0_3.2_1661340138139.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legpipe_whereas_en_1.0.0_3.2_1661340138139.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from johnsnowlabs import * deid_pipeline = PretrainedPipeline("legpipe_whereas", "en", "legal/models") deid_pipeline.annotate('WHEREAS VerticalNet owns and operates a series of online communities.') # Return NER chunks pipeline_result['ner_chunk'] # Return RE pipeline_result['relations'] # Visualize the Dependencies dependency_vis = viz.DependencyParserVisualizer() dependency_vis.display(pipeline_result[0], #should be the results of a single example, not the complete dataframe. pos_col = 'pos', #specify the pos column dependency_col = 'dependencies', #specify the dependency column dependency_type_col = 'dependency_type' #specify the dependency type column ) ```

## Results ```bash # NER ['VerticalNet', 'operates', 'a series of online communities'] # Relations ['has_subject', 'has_subject', 'has_object'] # DEP # Use Spark NLP Display to see the dependency tree ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legpipe_whereas| |Type:|pipeline| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|918.6 MB| ## References In-house annotations on CUAD dataset ## Included Models - nlp.DocumentAssembler - nlp.Tokenizer - nlp.PerceptronModel - nlp.DependencyParserModel - nlp.TypedDependencyParserModel - nlp.RoBertaEmbeddings - legal.NerModel - nlp.NerConverter - legal.RelationExtractionDLModel --- layout: model title: English BertForQuestionAnswering Base Cased model (from roshnir) author: John Snow Labs name: bert_qa_base_multi_mlqa_dev date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multi-mlqa-dev-en` is a English model originally trained by `roshnir`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_base_multi_mlqa_dev_en_4.0.0_3.0_1657183101705.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_base_multi_mlqa_dev_en_4.0.0_3.0_1657183101705.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_base_multi_mlqa_dev","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_base_multi_mlqa_dev","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_base_multi_mlqa_dev| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|626.2 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/roshnir/bert-base-multi-mlqa-dev-en --- layout: model title: Detect Clinical Events author: John Snow Labs name: ner_events_clinical date: 2021-03-31 tags: [ner, clinical, licensed, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 supported: true annotator: MedicalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model can be used to detect clinical events in medical text. ## Predicted Entities `DATE`, `TIME`, `PROBLEM`, `TEST`, `TREATMENT`, `OCCURENCE`, `CLINICAL_DEPT`, `EVIDENTIAL`, `DURATION`, `FREQUENCY`, `ADMISSION`, `DISCHARGE`. {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/NER_EVENTS_CLINICAL/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_events_clinical_en_3.0.0_3.0_1617209685283.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_events_clinical_en_3.0.0_3.0_1617209685283.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\ .setInputCols(["sentence", "token"])\ .setOutputCol("embeddings") clinical_ner = MedicalNerModel.pretrained("ner_events_clinical", "en", "clinical/models") \ .setInputCols(["sentence", "token", "embeddings"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter]) model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) results = model.transform(spark.createDataFrame([["The patient presented to the emergency room last evening"]], ["text"])) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_detector = new SentenceDetector() .setInputCols("document") .setOutputCol("sentence") val tokenizer = new Tokenizer() .setInputCols("sentence") .setOutputCol("token") val word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token")) .setOutputCol("embeddings") val ner = MedicalNerModel.pretrained("ner_events_clinical", "en", "clinical/models") .setInputCols(Array("sentence", "token", "embeddings")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("sentence", "token", "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter)) val data = Seq("""The patient presented to the emergency room last evening""").toDS().toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.events_clinical").predict("""The patient presented to the emergency room last evening""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_embeddings_litlat_bert| |Compatibility:|Spark NLP 3.4.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|en| |Size:|361.9 MB| |Case sensitive:|true| ## References - https://huggingface.co/EMBEDDIA/litlat-bert --- layout: model title: Spanish RobertaForQuestionAnswering Base Cased model (from BSC-TeMU) author: John Snow Labs name: roberta_qa_bsc_temu_base_bne_s_c date: 2022-12-02 tags: [es, open_source, roberta, question_answering, tensorflow] task: Question Answering language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-bne-sqac` is a Spanish model originally trained by `BSC-TeMU`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_bsc_temu_base_bne_s_c_es_4.2.4_3.0_1669985795826.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_bsc_temu_base_bne_s_c_es_4.2.4_3.0_1669985795826.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bsc_temu_base_bne_s_c","es")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_bsc_temu_base_bne_s_c","es") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_bsc_temu_base_bne_s_c| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|es| |Size:|460.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/BSC-TeMU/roberta-base-bne-sqac - https://arxiv.org/abs/1907.11692 - http://www.bne.es/en/Inicio/index.html - https://github.com/PlanTL-SANIDAD/lm-spanish - https://arxiv.org/abs/2107.07253 --- layout: model title: Sentence Entity Resolver for Snomed Concepts, CT version (``sbiobert_base_cased_mli`` embeddings) author: John Snow Labs name: sbiobertresolve_snomed_findings date: 2021-05-16 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.4 spark_version: 3.0 supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps extracted medical entities to Snomed codes (CT version) using `sbiobert_base_cased_mli` Sentence Bert Embeddings, and has faster load time, with a speedup of about 6X when compared to previous versions. Also the load process now is more memory friendly meaning that the maximum memory required during load time is smaller, reducing the chances of OOM exceptions, and thus relaxing hardware requirements. ## Predicted Entities Predicts Snomed Codes and their normalized definition for each chunk. {:.btn-box} [Live Demo](https://nlp.johnsnowlabs.com/demos){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_en_3.0.4_3.0_1621191323188.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_snomed_findings_en_3.0.4_3.0_1621191323188.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use ```sbiobertresolve_snomed_findings``` resolver model must be used with ```sbiobert_base_cased_mli``` as embeddings ```ner_clinical``` as NER model. No need to set ```.setWhiteList()```.

## Results ```bash +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| resolutions| codes| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM| 38341003| 0.3234|hypertension:::hy...|38341003:::155295...| |chronic renal ins...| 83|109| PROBLEM|723190009| 0.7522|chronic renal ins...|723190009:::70904...| | COPD| 113|116| PROBLEM| 13645005| 0.1226|copd - chronic ob...|13645005:::155565...| | gastritis| 120|128| PROBLEM|235653009| 0.2444|gastritis:::gastr...|235653009:::45560...| | TIA| 136|138| PROBLEM|275382005| 0.0766|cerebral trauma (...|275382005:::44739...| |a non-ST elevatio...| 182|202| PROBLEM|233843008| 0.2224|silent myocardial...|233843008:::19479...| |Guaiac positive s...| 208|229| PROBLEM| 59614000| 0.9678|guaiac-positive s...|59614000:::703960...| |cardiac catheteri...| 295|317| TEST|301095005| 0.2584|cardiac finding::...|301095005:::25090...| | PTCA| 324|327|TREATMENT|373108000| 0.0809|post percutaneous...|373108000:::25103...| | mid LAD lesion| 332|345| PROBLEM|449567000| 0.0900|overriding left v...|449567000:::46140...| +--------------------+-----+---+---------+---------+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_snomed_findings| |Compatibility:|Healthcare NLP 3.0.4+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk, sbert_embeddings]| |Output Labels:|[snomed_ct_code]| |Language:|en| |Case sensitive:|false| ## Data Source Trained on SNOMED (CT version) Findings with ``sbiobert_base_cased_mli`` sentence embeddings. http://www.snomed.org/ --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from akdeniz27) author: John Snow Labs name: roberta_qa_akdeniz27_base_cuad date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-cuad` is a English model originally trained by `akdeniz27`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_akdeniz27_base_cuad_en_4.2.4_3.0_1669986180302.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_akdeniz27_base_cuad_en_4.2.4_3.0_1669986180302.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_akdeniz27_base_cuad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_akdeniz27_base_cuad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_akdeniz27_base_cuad| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|447.9 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/akdeniz27/roberta-base-cuad - https://github.com/TheAtticusProject/cuad - https://github.com/marshmellow77/cuad-demo --- layout: model title: Relation Extraction Between Body Parts and Procedures author: John Snow Labs name: redl_bodypart_procedure_test_biobert date: 2021-06-01 tags: [licensed, en, relation_extraction, clinical] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.0.3 spark_version: 3.0 supported: true annotator: RelationExtractionDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Relation extraction between body parts entities like ‘Internal_organ_or_component’, ’External_body_part_or_region’ etc. and procedure and test entities. `1` : body part and test/procedure are related to each other. `0` : body part and test/procedure are not related to each other. ## Predicted Entities `0`, `1` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.1.Clinical_Relation_Extraction_BodyParts_Models.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_3.0.3_3.0_1622581871045.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/redl_bodypart_procedure_test_biobert_en_3.0.3_3.0_1622581871045.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash | | relation | entity1 | chunk1 | entity2 | chunk2 | confidence | |---:|-----------:|:-----------------------------|:---------|:----------|:--------------------|-------------:| | 0 | 1 | External_body_part_or_region | chest | Test | portable ultrasound | 0.99953 | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|redl_bodypart_procedure_test_biobert| |Compatibility:|Healthcare NLP 3.0.3+| |License:|Licensed| |Edition:|Official| |Language:|en| |Case sensitive:|true| ## Data Source Trained on a custom internal dataset. ## Benchmarking ```bash Relation Recall Precision F1 Support 0 0.338 0.472 0.394 325 1 0.904 0.843 0.872 1275 Avg. 0.621 0.657 0.633 - ``` --- layout: model title: Legal Enforceability Clause Binary Classifier (LEDGAR) author: John Snow Labs name: legclf_enforceability_bert date: 2023-03-05 tags: [en, legal, classification, clauses, enforceability, licensed, tensorflow] task: Text Classification language: en edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description LEDGAR dataset aims to contract provision (paragraph) classification. The contract provisions come from contracts obtained from the US Securities and Exchange Commission (SEC) filings, which are publicly available from EDGAR. Each label represents the single main topic (theme) of the corresponding contract provision. This model is a Binary Classifier (True, False) for the `Enforceability` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/01.Page_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `Enforceability`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_enforceability_bert_en_1.0.0_3.0_1678047282639.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_enforceability_bert_en_1.0.0_3.0_1678047282639.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_enforceability_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[Enforceability]| |[Other]| |[Other]| |[Enforceability]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_enforceability_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.5 MB| ## References Train dataset available [here](https://huggingface.co/datasets/lex_glue) ## Benchmarking ```bash label precision recall f1-score support Enforceability 0.78 0.84 0.81 25 Other 0.90 0.86 0.88 43 accuracy - - 0.85 68 macro-avg 0.84 0.85 0.84 68 weighted-avg 0.86 0.85 0.85 68 ``` --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from csarron) author: John Snow Labs name: roberta_qa_base_squad_v1 date: 2022-12-02 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad-v1` is a English model originally trained by `csarron`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad_v1_en_4.2.4_3.0_1669986599451.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_base_squad_v1_en_4.2.4_3.0_1669986599451.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad_v1","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_base_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_base_squad_v1| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|457.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/csarron/roberta-base-squad-v1 - https://arxiv.org/abs/1907.11692 - https://rajpurkar.github.io/SQuAD-explorer - https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json - https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json - https://awk.ai/ - https://github.com/csarron - https://twitter.com/sysnlp --- layout: model title: Legal No assignment Clause Binary Classifier author: John Snow Labs name: legclf_no_assignment_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `no-assignment` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `no-assignment` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_no_assignment_clause_en_1.0.0_3.2_1660123750879.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_no_assignment_clause_en_1.0.0_3.2_1660123750879.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_no_assignment_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[no-assignment]| |[other]| |[other]| |[no-assignment]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_no_assignment_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support no-assignment 0.87 0.91 0.89 43 other 0.95 0.93 0.94 89 accuracy - - 0.92 132 macro-avg 0.91 0.92 0.91 132 weighted-avg 0.93 0.92 0.92 132 ``` --- layout: model title: Legal Investment Advisory And Management Agreement Document Classifier (Bert Sentence Embeddings) author: John Snow Labs name: legclf_investment_advisory_and_management_agreement_bert date: 2023-01-29 tags: [en, legal, classification, investment, advisory, management, agreement, licensed, bert, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_investment_advisory_and_management_agreement_bert` model is a Bert Sentence Embeddings Document Classifier used to classify if the document belongs to the class `investment-advisory-and-management-agreement` or not (Binary Classification). Unlike the Longformer model, this model is lighter in terms of inference time. ## Predicted Entities `investment-advisory-and-management-agreement`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_investment_advisory_and_management_agreement_bert_en_1.0.0_3.0_1674990433150.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_investment_advisory_and_management_agreement_bert_en_1.0.0_3.0_1674990433150.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_investment_advisory_and_management_agreement_bert", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[investment-advisory-and-management-agreement]| |[other]| |[other]| |[investment-advisory-and-management-agreement]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_investment_advisory_and_management_agreement_bert| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.2 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support investment-advisory-and-management-agreement 1.00 0.97 0.99 34 other 0.98 1.00 0.99 53 accuracy - - 0.99 87 macro-avg 0.99 0.99 0.99 87 weighted-avg 0.99 0.99 0.99 87 ``` --- layout: model title: Legal Right Of First Refusal Clause Binary Classifier author: John Snow Labs name: legclf_right_of_first_refusal_clause date: 2023-01-29 tags: [en, legal, classification, right, first, refusal, clauses, right_of_first_refusal, licensed, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow annotator: LegalClassifierDLModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `right-of-first-refusal` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `right-of-first-refusal`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_right_of_first_refusal_clause_en_1.0.0_3.0_1675005454414.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_right_of_first_refusal_clause_en_1.0.0_3.0_1675005454414.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en")\ .setInputCols("document")\ .setOutputCol("sentence_embeddings") doc_classifier = legal.ClassifierDLModel.pretrained("legclf_right_of_first_refusal_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ document_assembler, embeddings, doc_classifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ |result| +-------+ |[right-of-first-refusal]| |[other]| |[other]| |[right-of-first-refusal]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_right_of_first_refusal_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|22.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 1.00 0.97 0.99 38 right-of-first-refusal 0.96 1.00 0.98 22 accuracy - - 0.98 60 macro-avg 0.98 0.99 0.98 60 weighted-avg 0.98 0.98 0.98 60 ``` --- layout: model title: Stopwords Remover for Albanian language (223 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, sq, open_source] task: Stop Words Removal language: sq edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_sq_3.4.1_3.0_1646673139987.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_sq_3.4.1_3.0_1646673139987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+ |result | +-------------------------+ |[jeni, më, të, mirë, unë]| +-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|sq| |Size:|1.9 KB| --- layout: model title: English RobertaForQuestionAnswering Cased model (from AnonymousSub) author: John Snow Labs name: roberta_qa_emanuals_squad2.0 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `EManuals_RoBERTa_squad2.0` is a English model originally trained by `AnonymousSub`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_emanuals_squad2.0_en_4.3.0_3.0_1674208019986.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_emanuals_squad2.0_en_4.3.0_3.0_1674208019986.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_emanuals_squad2.0","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_emanuals_squad2.0","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_emanuals_squad2.0| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|466.7 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/AnonymousSub/EManuals_RoBERTa_squad2.0 --- layout: model title: Abkhazian asr_speech_sprint_test TFWav2Vec2ForCTC from Mofe author: John Snow Labs name: asr_speech_sprint_test date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_speech_sprint_test` is a Abkhazian model originally trained by Mofe. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_speech_sprint_test_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_speech_sprint_test_ab_4.2.0_3.0_1664021394571.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_speech_sprint_test_ab_4.2.0_3.0_1664021394571.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_speech_sprint_test", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_speech_sprint_test", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_speech_sprint_test| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|446.9 KB| --- layout: model title: Relation extraction between Drugs and ADE - Conversational Text author: John Snow Labs name: re_ade_conversational date: 2022-07-27 tags: [relation_extraction, licensed, clinical, en] task: Relation Extraction language: en nav_key: models edition: Healthcare NLP 3.5.0 spark_version: 3.0 supported: true annotator: RelationExtractionModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is capable of Relating Drugs and adverse reactions caused by them in conversational text. ## Predicted Entities `is_related`, `not_related` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/RE_ADE/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/10.Clinical_Relation_Extraction.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/re_ade_conversational_en_3.5.0_3.0_1658956087191.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/re_ade_conversational_en_3.5.0_3.0_1658956087191.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documenter = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("sentences") tokenizer = Tokenizer()\ .setInputCols(["sentences"])\ .setOutputCol("tokens") words_embedder = WordEmbeddingsModel() \ .pretrained("embeddings_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"]) \ .setOutputCol("embeddings") ner_tagger = MedicalNerModel() \ .pretrained("ner_ade_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens", "embeddings"]) \ .setOutputCol("ner_tags") ner_converter = NerConverter() \ .setInputCols(["sentences", "tokens", "ner_tags"]) \ .setOutputCol("ner_chunks") pos_tagger = PerceptronModel()\ .pretrained("pos_clinical", "en", "clinical/models") \ .setInputCols(["sentences", "tokens"])\ .setOutputCol("pos_tags") dependency_parser = sparknlp.annotators.DependencyParserModel()\ .pretrained("dependency_conllu", "en")\ .setInputCols(["sentences", "pos_tags", "tokens"])\ .setOutputCol("dependencies") re_model = RelationExtractionModel()\ .pretrained("re_ade_conversational", "en", "clinical/models")\ .setInputCols(["embeddings", "pos_tags", "ner_chunks", "dependencies"])\ .setOutputCol("relations")\ .setRelationPairs(["ade-drug", "drug-ade"]) # Possible relation pairs. Default: All Relations. nlp_pipeline = Pipeline(stages=[documenter, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_converter, dependency_parser, re_model]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text"))) text ="""19.32 day 20 rivaroxaban diary. still residual aches and pains; only had 4 paracetamol today.""" annotations = light_pipeline.fullAnnotate(text) ``` ```scala val documenter = new DocumentAssembler() .setInputCol("text") .setOutputCol("sentences") val tokenizer = new Tokenizer() .setInputCols("sentences") .setOutputCol("tokens") val words_embedder = WordEmbeddingsModel() .pretrained("embeddings_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("embeddings") val ner_tagger = NerDLModel() .pretrained("ner_ade_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens", "embeddings")) .setOutputCol("ner_tags") val ner_converter = new NerConverter() .setInputCols(Array("sentences", "tokens", "ner_tags")) .setOutputCol("ner_chunks") val pos_tagger = PerceptronModel() .pretrained("pos_clinical", "en", "clinical/models") .setInputCols(Array("sentences", "tokens")) .setOutputCol("pos_tags") val dependency_parser = DependencyParserModel() .pretrained("dependency_conllu", "en") .setInputCols(Array("sentences", "pos_tags", "tokens")) .setOutputCol("dependencies") val re_model = RelationExtractionModel() .pretrained("re_ade_conversational", "en", "clinical/models") .setInputCols(Array("embeddings", "pos_tags", "ner_chunks", "dependencies")) .setOutputCol("relations") .setMaxSyntacticDistance(3) #default: 0 .setPredictionThreshold(0.5) #default: 0.5 .setRelationPairs(Array("drug-ade", "ade-drug")) # Possible relation pairs. Default: All Relations. val nlpPipeline = new Pipeline().setStages(Array(documenter, tokenizer, words_embedder, pos_tagger, ner_tagger, ner_chunker, dependency_parser, re_model)) val data = Seq("""19.32 day 20 rivaroxaban diary. still residual aches and pains; only had 4 paracetamol today.""").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.relation.adverse_drug_events.conversational").predict("""19.32 day 20 rivaroxaban diary. still residual aches and pains; only had 4 paracetamol today.""") ```

## Results ```bash | | chunk1 | entitiy1 | chunk2 | entity2 | relation | |----|-------------------------------|------------|-------------|---------|-------------| | 0 | residual aches and pains | ADE | rivaroxaban | DRUG | is_related | | 1 | residual aches and pains | ADE | paracetamol | DRUG | not_related | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|re_ade_conversational| |Type:|re| |Compatibility:|Healthcare NLP 3.5.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[embeddings, pos_tags, train_ner_chunks, dependencies]| |Output Labels:|[relations]| |Language:|en| |Size:|11.3 MB| ## References Trained on SMM4H dataset - annotated manually. https://healthlanguageprocessing.org/smm4h-2022/ ## Benchmarking ```bash label precision recall f1-score support not_related 0.81 0.88 0.85 528 is_related 0.94 0.89 0.91 1019 accuracy - - 0.89 1547 macro-avg 0.87 0.89 0.88 1547 weighted-avg 0.89 0.89 0.89 1547 ``` --- layout: model title: Fast Neural Machine Translation Model from Tuvaluan to English author: John Snow Labs name: opus_mt_tvl_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, tvl, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `tvl` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_tvl_en_xx_2.7.0_2.4_1609163187137.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_tvl_en_xx_2.7.0_2.4_1609163187137.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_tvl_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Persian BertForQuestionAnswering Cased model (from marzinouri101) author: John Snow Labs name: bert_qa_parsbert_finetuned_persianqa date: 2022-07-07 tags: [fa, open_source, bert, question_answering] task: Question Answering language: fa edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `parsbert-finetuned-persianQA` is a Persian model originally trained by `marzinouri101`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_parsbert_finetuned_persianqa_fa_4.0.0_3.0_1657190740642.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_parsbert_finetuned_persianqa_fa_4.0.0_3.0_1657190740642.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_parsbert_finetuned_persianqa","fa") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_parsbert_finetuned_persianqa","fa") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("اسم من چیست؟", "نام من کلارا است و من در برکلی زندگی می کنم.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_parsbert_finetuned_persianqa| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|fa| |Size:|442.3 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/marzinouri101/parsbert-finetuned-persianQA --- layout: model title: Spanish Legal Roberta Embeddings author: John Snow Labs name: roberta_base_spanish_legal date: 2023-02-16 tags: [es, spanish, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: es edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-spanish-roberta-base` is a Spanish model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_spanish_legal_es_4.2.4_3.0_1676579126608.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_spanish_legal_es_4.2.4_3.0_1676579126608.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_spanish_legal", "es")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_spanish_legal", "es") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_spanish_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|416.2 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-spanish-roberta-base --- layout: model title: Sentence Entity Resolver for CPT (sbiobert_base_cased_mli embeddings) author: John Snow Labs name: sbiobertresolve_cpt language: en nav_key: models repository: clinical/models date: 2020-11-27 task: Entity Resolution edition: Healthcare NLP 2.6.4 spark_version: 2.4 tags: [clinical,entity_resolution,en] supported: true annotator: SentenceEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model maps extracted medical entities to CPT codes using chunk embeddings. {:.h2_title} ## Predicted Entities CPT Codes and their normalized definition with ``sbiobert_base_cased_mli`` sentence embeddings. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/24.Improved_Entity_Resolvers_in_SparkNLP_with_sBert.ipynb){:.button.button-orange.button-orange-trans.co.button-icon}{:target="_blank"} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_en_2.6.4_2.4_1606235767322.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_en_2.6.4_2.4_1606235767322.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") sbert_embedder = BertSentenceEmbeddings\ .pretrained("sbiobert_base_cased_mli","en","clinical/models")\ .setInputCols(["ner_chunk_doc"])\ .setOutputCol("sbert_embeddings") cpt_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_cpt","en", "clinical/models") \ .setInputCols(["sbert_embeddings"]) \ .setOutputCol("resolution")\ .setDistanceFunction("EUCLIDEAN") nlpPipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, cpt_resolver]) data = spark.createDataFrame([["This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU ."]]).toDF("text") results = nlpPipeline.fit(data).transform(data) ``` ```scala ... chunk2doc = Chunk2Doc().setInputCols("ner_chunk").setOutputCol("ner_chunk_doc") val sbert_embedder = BertSentenceEmbeddings .pretrained("sbiobert_base_cased_mli","en","clinical/models") .setInputCols(Array("ner_chunk_doc")) .setOutputCol("sbert_embeddings") val cpt_resolver = SentenceEntityResolverModel .pretrained("sbiobertresolve_cpt","en", "clinical/models") .setInputCols(Array("ner_chunk", "sbert_embeddings")) .setOutputCol("resolution") .setDistanceFunction("EUCLIDEAN") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk2doc, sbert_embedder, cpt_resolver)) val data = Seq("This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD , gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\'s Center for Women & Infants for cardiac catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.h2_title} ## Results ```bash +--------------------+-----+---+---------+-----+----------+--------------------+--------------------+ | chunk|begin|end| entity| code|confidence| all_k_resolutions| all_k_codes| +--------------------+-----+---+---------+-----+----------+--------------------+--------------------+ | hypertension| 68| 79| PROBLEM|49425| 0.0967|Insertion of peri...|49425:::36818:::3...| |chronic renal ins...| 83|109| PROBLEM|50070| 0.2569|Nephrolithotomy; ...|50070:::49425:::5...| | COPD| 113|116| PROBLEM|49425| 0.0779|Insertion of peri...|49425:::31592:::4...| | gastritis| 120|128| PROBLEM|43810| 0.5289|Gastroduodenostom...|43810:::43880:::4...| | TIA| 136|138| PROBLEM|25927| 0.2060|Transmetacarpal a...|25927:::25931:::6...| |a non-ST elevatio...| 182|202| PROBLEM|33300| 0.3046|Repair of cardiac...|33300:::33813:::3...| |Guaiac positive s...| 208|229| PROBLEM|47765| 0.0974|Anastomosis, of i...|47765:::49425:::1...| |cardiac catheteri...| 295|317| TEST|62225| 0.1996|Replacement or ir...|62225:::33722:::4...| | PTCA| 324|327|TREATMENT|60500| 0.1481|Parathyroidectomy...|60500:::43800:::2...| | mid LAD lesion| 332|345| PROBLEM|33722| 0.3097|Closure of aortic...|33722:::33732:::3...| +--------------------+-----+---+---------+-----+----------+--------------------+--------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---------------|---------------------| | Name: | sbiobertresolve_cpt | | Type: | SentenceEntityResolverModel | | Compatibility: | Spark NLP 2.6.4 + | | License: | Licensed | | Edition: | Official | |Input labels: | [ner_chunk, chunk_embeddings] | |Output labels: | [resolution] | | Language: | en | | Dependencies: | sbiobert_base_cased_mli | {:.h2_title} ## Data Source Trained on Current Procedural Terminology dataset with ``sbiobert_base_cased_mli`` sentence embeddings. --- layout: model title: Translate Tok Pisin to English Pipeline author: John Snow Labs name: translate_tpi_en date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, tpi, en, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `tpi` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_tpi_en_xx_2.7.0_2.4_1609689664783.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_tpi_en_xx_2.7.0_2.4_1609689664783.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_tpi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_tpi_en", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.tpi.translate_to.en').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_tpi_en| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English RobertaForQuestionAnswering Cased model (from eAsyle) author: John Snow Labs name: roberta_qa_testabsa3 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `testABSA3` is a English model originally trained by `eAsyle`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_testabsa3_en_4.3.0_3.0_1674224346874.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_testabsa3_en_4.3.0_3.0_1674224346874.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_testabsa3","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_testabsa3","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_testabsa3| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|426.2 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/eAsyle/testABSA3 --- layout: model title: Polish BertForQuestionAnswering model (from henryk) author: John Snow Labs name: bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1 date: 2022-06-02 tags: [pl, open_source, question_answering, bert] task: Question Answering language: pl edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-multilingual-cased-finetuned-polish-squad1` is a Polish model orginally trained by `henryk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1_pl_4.0.0_3.0_1654180093762.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1_pl_4.0.0_3.0_1654180093762.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1","pl") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = BertForQuestionAnswering .pretrained("bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1","pl") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("pl.answer_question.squad.bert.multilingual_base_cased").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_bert_base_multilingual_cased_finetuned_polish_squad1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|pl| |Size:|665.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/henryk/bert-base-multilingual-cased-finetuned-polish-squad1 - https://www.linkedin.com/in/henryk-borzymowski-0755a2167/ - https://rajpurkar.github.io/SQuAD-explorer/ - https://github.com/google-research/bert/blob/master/multilingual.md --- layout: model title: Financial Security ownership Item Binary Classifier author: John Snow Labs name: finclf_security_ownership_item date: 2022-08-10 tags: [en, finance, classification, 10k, annual, reports, sec, filings, licensed] task: Text Classification language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `security_ownership` item type of 10K Annual Reports. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big financial documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Finance NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Finance/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). ## Predicted Entities `other`, `security_ownership` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finclf_security_ownership_item_en_1.0.0_3.2_1660154472575.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finclf_security_ownership_item_en_1.0.0_3.2_1660154472575.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("document") useEmbeddings = nlp.UniversalSentenceEncoder.pretrained() \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("finclf_security_ownership_item", "en", "finance/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, useEmbeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[security_ownership]| |[other]| |[other]| |[security_ownership]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finclf_security_ownership_item| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|22.6 MB| ## References Weak labelling on documents from Edgar database ## Benchmarking ```bash label precision recall f1-score support other 0.90 0.84 0.87 31 security_ownership 0.78 0.86 0.82 21 accuracy - - 0.85 52 macro-avg 0.84 0.85 0.84 52 weighted-avg 0.85 0.85 0.85 52 ``` --- layout: model title: Word2Vec Embeddings in Mongolian (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-16 tags: [cc, embeddings, fastText, word2vec, mn, open_source] task: Embeddings language: mn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_mn_3.4.1_3.0_1647447849598.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_mn_3.4.1_3.0_1647447849598.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_wl_base","es") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, embeddings]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val embeddings = DeBertaEmbeddings.pretrained("deberta_embeddings_wl_base","es") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, embeddings)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|deberta_embeddings_wl_base| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|es| |Size:|1.0 GB| |Case sensitive:|false| ## References https://huggingface.co/plncmm/mdeberta-wl-base-es --- layout: model title: Fast Neural Machine Translation Model from Azerbaijani to Spanish author: John Snow Labs name: opus_mt_az_es date: 2021-06-01 tags: [open_source, seq2seq, translation, az, es, xx, multilingual] task: Translation language: xx edition: Spark NLP 3.1.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). source languages: az target languages: es {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_az_es_xx_3.1.0_2.4_1622560735224.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_az_es_xx_3.1.0_2.4_1622560735224.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_az_es| |Compatibility:|Spark NLP 3.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Lemmatizer (Spanish, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, es] task: Lemmatization language: es edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Spanish Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_es_3.4.1_3.0_1646316590883.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_es_3.4.1_3.0_1646316590883.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+ |result | +-------------------------+ |[No, ser, mejor, que, yo]| +-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|es| |Size:|5.3 MB| --- layout: model title: BioBERT Sentence Embeddings (Pubmed PMC) author: John Snow Labs name: sent_biobert_pubmed_pmc_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertSentenceEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of BioBERT, a language representation model for biomedical domain, especially designed for biomedical text mining tasks such as biomedical named entity recognition, relation extraction, question answering, etc. The details are described in the paper "[BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://arxiv.org/abs/1901.08746)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/sent_biobert_pubmed_pmc_base_cased_en_2.6.0_2.4_1598349155555.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/sent_biobert_pubmed_pmc_base_cased_en_2.6.0_2.4_1598349155555.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pubmed_pmc_base_cased", "en") \ .setInputCols("sentence") \ .setOutputCol("sentence_embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I hate cancer', "Antibiotics aren't painkiller"]], ["text"])) ``` ```scala ... val embeddings = BertSentenceEmbeddings.pretrained("sent_biobert_pubmed_pmc_base_cased", "en") .setInputCols("sentence") .setOutputCol("sentence_embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, embeddings)) val data = Seq("I hate cancer, "Antibiotics aren't painkiller").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I hate cancer", "Antibiotics aren't painkiller"] embeddings_df = nlu.load('en.embed_sentence.biobert.pubmed_pmc_base_cased').predict(text, output_level='sentence') embeddings_df ```

{:.h2_title} ## Results ```bash sentence en_embed_sentence_biobert_pubmed_pmc_base_cased_embeddings I hate cancer [0.2354733943939209, 0.30127033591270447, -0.1... Antibiotics aren't painkiller [0.2837969958782196, 0.03842488303780556, 0.04... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sent_biobert_pubmed_pmc_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[sentence_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/dmis-lab/biobert](https://github.com/dmis-lab/biobert) --- layout: model title: Moldavian, Moldovan, Romanian asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila TFWav2Vec2ForCTC from gmihaila author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila date: 2022-09-25 tags: [wav2vec2, ro, audio, open_source, asr] task: Automatic Speech Recognition language: ro edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila` is a Moldavian, Moldovan, Romanian model originally trained by gmihaila. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila_ro_4.2.0_3.0_1664099376954.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila_ro_4.2.0_3.0_1664099376954.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila", "ro")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila", "ro") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_romanian_by_gmihaila| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ro| |Size:|1.2 GB| --- layout: model title: ICD10CM Injuries Entity Resolver author: John Snow Labs name: chunkresolve_icd10cm_injuries_clinical date: 2021-04-02 tags: [entity_resolution, clinical, licensed, en] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.0.0 spark_version: 3.0 deprecated: true annotator: ChunkEntityResolverModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Entity Resolution model Based on KNN using Word Embeddings + Word Movers Distance ## Predicted Entities ICD10-CM Codes and their normalized definition with `clinical_embeddings` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_ICD10_CM/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_injuries_clinical_en_3.0.0_3.0_1617355437876.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/chunkresolve_icd10cm_injuries_clinical_en_3.0.0_3.0_1617355437876.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... injury_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_injuries_clinical","en","clinical/models")\ .setInputCols("token","chunk_embeddings")\ .setOutputCol("entity") pipeline_puerile = Pipeline(stages = [documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, injury_resolver]) model = pipeline_puerile.fit(spark.createDataFrame([["""The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion."""]]).toDF("text")) results = model.transform(data) ``` ```scala ... val injury_resolver = ChunkEntityResolverModel.pretrained("chunkresolve_icd10cm_injuries_clinical","en","clinical/models") .setInputCols(Array("token","chunk_embeddings")) .setOutputCol("resolution") val pipeline = new Pipeline().setStages(Array(documentAssembler, sentenceDetector, tokenizer, word_embeddings, clinical_ner, ner_converter, chunk_embeddings, injury_resolver)) val data = Seq("The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.").toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash chunk entity icd10_inj_description icd10_inj_code 0 a cold, cough PROBLEM Insect bite (nonvenomous) of throat, sequela S1016XS 1 runny nose PROBLEM Abrasion of nose, initial encounter S0031XA 2 fever PROBLEM Blister (nonthermal) of abdominal wall, initia... S30821A 3 difficulty breathing PROBLEM Concussion without loss of consciousness, init... S060X0A 4 her cough PROBLEM Contusion of throat, subsequent encounter S100XXD 5 physical exam TEST Concussion without loss of consciousness, init... S060X0A 6 fairly congested PROBLEM Contusion and laceration of right cerebrum wit... S06310A 7 Amoxil TREATMENT Insect bite (nonvenomous), unspecified ankle, ... S90569S 8 Aldex TREATMENT Insect bite (nonvenomous) of penis, initial en... S30862A 9 difficulty breathing PROBLEM Concussion without loss of consciousness, init... S060X0A 10 more congested PROBLEM Complete traumatic amputation of two or more r... S98211S 11 trouble sleeping PROBLEM Concussion without loss of consciousness, sequela S060X0S 12 congestion PROBLEM Unspecified injury of portal vein, initial enc... S35319A ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|chunkresolve_icd10cm_injuries_clinical| |Compatibility:|Healthcare NLP 3.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[token, chunk_embeddings]| |Output Labels:|[icd10cm]| |Language:|en| --- layout: model title: Stopwords Remover for Urdu language (508 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, ur, open_source] task: Stop Words Removal language: ur edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_ur_3.4.1_3.0_1646672328433.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_ur_3.4.1_3.0_1646672328433.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------+ |result | +-------------------------+ |[تم, مجھ, سے, بہتر, نہیں]| +-------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|ur| |Size:|3.3 KB| --- layout: model title: Fast Neural Machine Translation Model from Venda to English author: John Snow Labs name: opus_mt_ve_en date: 2020-12-28 task: Translation language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, ve, en, xx] supported: true annotator: MarianTransformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). - source languages: `ve` - target languages: `en` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/opus_mt_ve_en_xx_2.7.0_2.4_1609169172425.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/opus_mt_ve_en_xx_2.7.0_2.4_1609169172425.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|opus_mt_ve_en| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Input Labels:|[sentence]| |Output Labels:|[translation]| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: English asr_wav2vec2_base_timit_demo_colab52 TFWav2Vec2ForCTC from hassnain author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab52 date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab52` is a English model originally trained by hassnain. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab52_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab52_en_4.2.0_3.0_1664022006865.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab52_en_4.2.0_3.0_1664022006865.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab52', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab52", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab52| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|355.0 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from mdineshk) author: John Snow Labs name: distilbert_qa_base_uncased_meded date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-meded` is a English model originally trained by `mdineshk`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_meded_en_4.3.0_3.0_1672773966077.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_base_uncased_meded_en_4.3.0_3.0_1672773966077.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_meded","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_base_uncased_meded","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_base_uncased_meded| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/mdineshk/distilbert-base-uncased-meded --- layout: model title: English asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman` is a English model originally trained by jonatasgrosman. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman_en_4.2.0_3.0_1664036112317.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman_en_4.2.0_3.0_1664036112317.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_wav2vec2_large_xlsr_53_english_by_jonatasgrosman| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|1.2 GB| --- layout: model title: Named Entity Recognition in Romanian Official Documents (Small) author: John Snow Labs name: legner_romanian_official_sm date: 2022-11-10 tags: [ro, ner, legal, licensed] task: Named Entity Recognition language: ro edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true annotator: LegalNerModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a small version of NER model that extracts only PER(Person), LOC(Location), ORG(Organization) and DATE entities from Romanian Official Documents. ## Predicted Entities `PER`, `LOC`, `ORG`, `DATE` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/legal/LEGNER_ROMANIAN_OFFICIAL/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_sm_ro_1.0.0_3.0_1668082337617.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legner_romanian_official_sm_ro_1.0.0_3.0_1668082337617.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document")\ sentence_detector = nlp.SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\ .setInputCols(["document"])\ .setOutputCol("sentence")\ tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_base_cased", "ro")\ .setInputCols("sentence", "token")\ .setOutputCol("embeddings")\ .setMaxSentenceLength(512)\ .setCaseSensitive(True) ner_model = legal.NerModel.pretrained("legner_romanian_official_sm", "ro", "legal/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = nlp.Pipeline(stages=[ document_assembler, sentence_detector, tokenizer, embeddings, ner_model, ner_converter ]) model = pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) data = spark.createDataFrame([["""Prezentul ordin se publică în Monitorul Oficial al României, Partea I. Ministrul sănătății, Sorina Pintea București, 28 februarie 2019.""]]).toDF("text") result = model.transform(data) ```

## Results ```bash +-----------------------------+-----+ |chunk |label| +-----------------------------+-----+ |Monitorul Oficial al României|ORG | |Sorina Pintea |PER | |București |LOC | |28 februarie 2019 |DATE | +-----------------------------+-----+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legner_romanian_official_sm| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence, token, embeddings]| |Output Labels:|[ner]| |Language:|ro| |Size:|16.4 MB| ## References Dataset is available [here](https://zenodo.org/record/7025333#.Y2zsquxBx83). ## Benchmarking ```bash label precision recall f1-score support DATE 0.87 0.96 0.91 397 LOC 0.87 0.78 0.83 190 ORG 0.90 0.93 0.91 559 PER 0.98 0.93 0.95 108 micro-avg 0.89 0.92 0.90 1254 macro-avg 0.91 0.90 0.90 1254 weighted-avg 0.89 0.92 0.90 1254 ``` --- layout: model title: Word2Vec Embeddings in Esperanto (300d) author: John Snow Labs name: w2v_cc_300d date: 2022-03-15 tags: [cc, embeddings, fastText, word2vec, eo, open_source] task: Embeddings language: eo edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Word Embeddings lookup annotator that maps tokens to vectors. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eo_3.4.1_3.0_1647368870513.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/w2v_cc_300d_eo_3.4.1_3.0_1647368870513.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", 'en','clinical/models')\ .setInputCols(["document"])\ .setOutputCol("sentence_embeddings") features_asm = FeaturesAssembler()\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("features") generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_insurance_coverage_sbiobert_cased_mli", 'en', 'clinical/models')\ .setInputCols(["features"])\ .setOutputCol("prediction") pipeline = Pipeline(stages=[ document_assembler, sentence_embeddings, features_asm, generic_classifier ]) text_list = ["The patient's Medicaid insurance is limited with some medicaitons.", "She is under good coverage Medicare insurance", "The patient has poor coverage of Private insurance", """Medical File for John Smith, Male, Age 42 Chief Complaint: Patient complains of nausea, vomiting, and shortness of breath. History of Present Illness: The patient has a history of hypertension and diabetes, which are both poorly controlled. The patient has been feeling unwell for the past week, with symptoms including nausea, vomiting, and shortness of breath. Upon examination, the patient was found to have a high serum creatinine level of 5.8 mg/dL, indicating renal failure. Past Medical History: The patient has a history of hypertension and diabetes, which have been poorly controlled due to poor medication adherence. The patient also has a history of smoking, which has been a contributing factor to the development of renal failure. Medications: The patient is currently taking Metformin and Lisinopril for the management of diabetes and hypertension, respectively. However, due to poor Medicaid coverage, the patient is unable to afford some of the medications prescribed by his physician. Insurance Status: The patient has Medicaid insurance, which provides poor coverage for some of the medications needed to manage his medical conditions, including those related to his renal failure. Physical Examination: Upon physical examination, the patient appears pale and lethargic. Blood pressure is 160/100 mmHg, heart rate is 90 beats per minute, and respiratory rate is 20 breaths per minute. There is diffuse abdominal tenderness on palpation, and lung auscultation reveals bilateral rales. Diagnosis: The patient is diagnosed with acute renal failure, likely due to uncontrolled hypertension and poorly managed diabetes. Treatment: The patient is started on intravenous fluids and insulin to manage his blood sugar levels. Due to the patient's poor Medicaid coverage, the physician works with the patient to identify alternative medications that are more affordable and will still provide effective management of his medical conditions. Follow-Up: The patient is advised to follow up with his primary care physician for ongoing management of his renal failure and other medical conditions. The patient is also referred to a nephrologist for further evaluation and management of his renal failure. """, """Certainly, here is an example case study for a patient with private insurance: Case Study for Emily Chen, Female, Age 38 Chief Complaint: Patient reports chronic joint pain and stiffness. History of Present Illness: The patient has been experiencing chronic joint pain and stiffness, particularly in the hands, knees, and ankles. The pain is worse in the morning and improves throughout the day. The patient has also noticed some swelling and redness in the affected joints. Past Medical History: The patient has a history of osteoarthritis, which has been gradually worsening over the past several years. The patient has tried over-the-counter pain relievers and joint supplements, but has not found significant relief. Medications: The patient is currently taking over-the-counter pain relievers and joint supplements for the management of her osteoarthritis. Insurance Status: The patient has private insurance, which provides comprehensive coverage for her medical care, including specialist visits and prescription medications. Physical Examination: Upon physical examination, the patient has tenderness and swelling in multiple joints, particularly the hands, knees, and ankles. Range of motion is limited due to pain and stiffness. Diagnosis: The patient is diagnosed with osteoarthritis, a chronic degenerative joint disease that causes pain, swelling, and stiffness in the affected joints. Treatment: The patient is prescribed a nonsteroidal anti-inflammatory drug (NSAID) to manage pain and inflammation. The physician also recommends physical therapy to improve range of motion and strengthen the affected joints. The patient is advised to continue taking joint supplements for ongoing joint health. Follow-Up: The patient is advised to follow up with the physician in 4-6 weeks to monitor response to treatment and make any necessary adjustments. The patient is also referred to a rheumatologist for further evaluation and management of her osteoarthritis."""] df = spark.createDataFrame(text_list, StringType()).toDF("text") result = pipeline.fit(df).transform(df) result.select("text", "prediction.result").show(truncate=100) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val sentence_embeddings = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en", "clinical/models") .setInputCols("document") .setOutputCol("sentence_embeddings") val features_asm = new FeaturesAssembler() .setInputCols("sentence_embeddings") .setOutputCol("features") val generic_classifier = GenericClassifierModel.pretrained("genericclassifier_sdoh_insurance_coverage_sbiobert_cased_mli", "en", "clinical/models") .setInputCols("features") .setOutputCol("prediction") val pipeline = new PipelineModel().setStages(Array( document_assembler, sentence_embeddings, features_asm, generic_classifier)) val data = Seq(Array("The patient's Medicaid insurance is limited with some medicaitons.", "She is under good coverage Medicare insurance", "The patient has poor coverage of Private insurance", """Medical File for John Smith, Male, Age 42 Chief Complaint: Patient complains of nausea, vomiting, and shortness of breath. History of Present Illness: The patient has a history of hypertension and diabetes, which are both poorly controlled. The patient has been feeling unwell for the past week, with symptoms including nausea, vomiting, and shortness of breath. Upon examination, the patient was found to have a high serum creatinine level of 5.8 mg/dL, indicating renal failure. Past Medical History: The patient has a history of hypertension and diabetes, which have been poorly controlled due to poor medication adherence. The patient also has a history of smoking, which has been a contributing factor to the development of renal failure. Medications: The patient is currently taking Metformin and Lisinopril for the management of diabetes and hypertension, respectively. However, due to poor Medicaid coverage, the patient is unable to afford some of the medications prescribed by his physician. Insurance Status: The patient has Medicaid insurance, which provides poor coverage for some of the medications needed to manage his medical conditions, including those related to his renal failure. Physical Examination: Upon physical examination, the patient appears pale and lethargic. Blood pressure is 160/100 mmHg, heart rate is 90 beats per minute, and respiratory rate is 20 breaths per minute. There is diffuse abdominal tenderness on palpation, and lung auscultation reveals bilateral rales. Diagnosis: The patient is diagnosed with acute renal failure, likely due to uncontrolled hypertension and poorly managed diabetes. Treatment: The patient is started on intravenous fluids and insulin to manage his blood sugar levels. Due to the patient's poor Medicaid coverage, the physician works with the patient to identify alternative medications that are more affordable and will still provide effective management of his medical conditions. Follow-Up: The patient is advised to follow up with his primary care physician for ongoing management of his renal failure and other medical conditions. The patient is also referred to a nephrologist for further evaluation and management of his renal failure. """, """Certainly, here is an example case study for a patient with private insurance: Case Study for Emily Chen, Female, Age 38 Chief Complaint: Patient reports chronic joint pain and stiffness. History of Present Illness: The patient has been experiencing chronic joint pain and stiffness, particularly in the hands, knees, and ankles. The pain is worse in the morning and improves throughout the day. The patient has also noticed some swelling and redness in the affected joints. Past Medical History: The patient has a history of osteoarthritis, which has been gradually worsening over the past several years. The patient has tried over-the-counter pain relievers and joint supplements, but has not found significant relief. Medications: The patient is currently taking over-the-counter pain relievers and joint supplements for the management of her osteoarthritis. Insurance Status: The patient has private insurance, which provides comprehensive coverage for her medical care, including specialist visits and prescription medications. Physical Examination: Upon physical examination, the patient has tenderness and swelling in multiple joints, particularly the hands, knees, and ankles. Range of motion is limited due to pain and stiffness. Diagnosis: The patient is diagnosed with osteoarthritis, a chronic degenerative joint disease that causes pain, swelling, and stiffness in the affected joints. Treatment: The patient is prescribed a nonsteroidal anti-inflammatory drug (NSAID) to manage pain and inflammation. The physician also recommends physical therapy to improve range of motion and strengthen the affected joints. The patient is advised to continue taking joint supplements for ongoing joint health. Follow-Up: The patient is advised to follow up with the physician in 4-6 weeks to monitor response to treatment and make any necessary adjustments. The patient is also referred to a rheumatologist for further evaluation and management of her osteoarthritis.""")).toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

## Results ```bash +----------------------------------------------------------------------------------------------------+------+ | text|result| +----------------------------------------------------------------------------------------------------+------+ | The patient's Medicaid insurance is limited with some medicaitons.|[Poor]| | She is under good coverage Medicare insurance|[Good]| | The patient has poor coverage of Private insurance|[Poor]| |Medical File for John Smith, Male, Age 42\n\nChief Complaint: Patient complains of nausea, vomiti...|[Poor]| |Certainly, here is an example case study for a patient with private insurance:\n\nCase Study for ...|[Good]| +----------------------------------------------------------------------------------------------------+------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|genericclassifier_sdoh_insurance_coverage_sbiobert_cased_mli| |Compatibility:|Healthcare NLP 4.4.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[features]| |Output Labels:|[prediction]| |Language:|en| |Size:|3.4 MB| |Dependencies:|sbiobert_base_cased_mli| ## References Internal SDOH project ## Benchmarking ```bash label precision recall f1-score support Good 0.81 0.86 0.84 74 Poor 0.94 0.84 0.89 70 Unknown 0.67 0.71 0.69 31 accuracy - - 0.83 175 macro-avg 0.80 0.81 0.80 175 weighted-avg 0.84 0.83 0.83 175 ``` --- layout: model title: English BertForQuestionAnswering Base Cased model (from anas-awadalla) author: John Snow Labs name: bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2 date: 2022-07-07 tags: [en, open_source, bert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-2` is a English model originally trained by `anas-awadalla`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2_en_4.0.0_3.0_1657192046030.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2_en_4.0.0_3.0_1657192046030.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_spanbert_base_cased_few_shot_k_16_finetuned_squad_seed_2| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|375.9 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/anas-awadalla/spanbert-base-cased-few-shot-k-16-finetuned-squad-seed-2 --- layout: model title: Smaller BERT Embeddings (L-4_H-512_A-8) author: John Snow Labs name: small_bert_L4_512 date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [open_source, embeddings, en] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is one of the smaller BERT models referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/small_bert_L4_512_en_2.6.0_2.4_1598344591466.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/small_bert_L4_512_en_2.6.0_2.4_1598344591466.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... embeddings = BertEmbeddings.pretrained("small_bert_L4_512", "en") \ .setInputCols("sentence", "token") \ .setOutputCol("embeddings") nlp_pipeline = Pipeline(stages=[document_assembler, sentence_detector, tokenizer, embeddings]) pipeline_model = nlp_pipeline.fit(spark.createDataFrame([[""]]).toDF("text")) result = pipeline_model.transform(spark.createDataFrame([['I love NLP']], ["text"])) ``` ```scala ... val embeddings = BertEmbeddings.pretrained("small_bert_L4_512", "en") .setInputCols("sentence", "token") .setOutputCol("embeddings") val pipeline = new Pipeline().setStages(Array(document_assembler, sentence_detector, tokenizer, embeddings)) val data = Seq("I love NLP").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["I love NLP"] embeddings_df = nlu.load('en.embed.bert.small_L4_512').predict(text, output_level='token') embeddings_df ```

{:.h2_title} ## Results ```bash token en_embed_bert_small_L4_512_embeddings I [0.5992571711540222, 0.4491763710975647, 0.515... love [0.9561082124710083, 0.11446993052959442, 0.19... NLP [-1.0831373929977417, 0.9501134753227234, -0.4... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|small_bert_L4_512| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|512| |Case sensitive:|false| {:.h2_title} ## Data Source The model is imported from https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1 --- layout: model title: Abkhazian asr_xls_r_ab_test_by_FitoDS TFWav2Vec2ForCTC from FitoDS author: John Snow Labs name: asr_xls_r_ab_test_by_FitoDS date: 2022-09-24 tags: [wav2vec2, ab, audio, open_source, asr] task: Automatic Speech Recognition language: ab edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_xls_r_ab_test_by_FitoDS` is a Abkhazian model originally trained by FitoDS. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_xls_r_ab_test_by_FitoDS_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_FitoDS_ab_4.2.0_3.0_1664021158373.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_xls_r_ab_test_by_FitoDS_ab_4.2.0_3.0_1664021158373.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_xls_r_ab_test_by_FitoDS", "ab")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_xls_r_ab_test_by_FitoDS", "ab") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_xls_r_ab_test_by_FitoDS| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|ab| |Size:|442.4 KB| --- layout: model title: Translate English to Basque Pipeline author: John Snow Labs name: translate_en_eu date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, eu, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `eu` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_eu_xx_2.7.0_2.4_1609691166839.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_eu_xx_2.7.0_2.4_1609691166839.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_eu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_eu", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.eu').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_eu| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Mapping National Drug Codes (NDC) Codes with Corresponding Drug Brand Names author: John Snow Labs name: ndc_drug_brandname_mapper date: 2023-02-22 tags: [chunk_mapping, ndc, drug_brand_name, clinical, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 4.3.0 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps National Drug Codes (NDC) codes with their corresponding drug brand names. ## Predicted Entities `drug_brand_name` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ndc_drug_brandname_mapper_en_4.3.0_3.0_1677102197072.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ndc_drug_brandname_mapper_en_4.3.0_3.0_1677102197072.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") mapper = DocMapperModel.pretrained("ndc_drug_brandname_mapper", "en", "clinical/models")\ .setInputCols("document")\ .setOutputCol("mappings")\ .setRels(["drug_brand_name"])\ pipeline = Pipeline( stages = [ documentAssembler, mapper ]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) result = lp.fullAnnotate(["0009-4992", "57894-150"]) ``` ```scala val documentAssembler = new DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") val mapper = DocMapperModel.pretrained("ndc_drug_brandname_mapper", "en", "clinical/models")\ .setInputCols("document")\ .setOutputCol("mappings")\ .setRels(Array("drug_brand_name")\ val pipeline = new Pipeline(stages = Array( documentAssembler, mapper )) val data = Seq(Array("0009-4992", "57894-150")).toDS.toDF("text") val result= pipeline.fit(data).transform(data) ```

## Results ```bash | | ndc_code | drug_brand_name | |---:|:-----------|:------------------| | 0 | 0009-4992 | ZYVOX | | 1 | 57894-150 | ZYTIGA | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ndc_drug_brandname_mapper| |Compatibility:|Healthcare NLP 4.3.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[brandname]| |Language:|en| |Size:|917.7 KB| --- layout: model title: Stopwords Remover for Bengali language (458 entries) author: John Snow Labs name: stopwords_iso date: 2022-03-07 tags: [stopwords, bn, open_source] task: Stop Words Removal language: bn edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This is a scalable, production-ready Stopwords Remover model trained using the corpus available at [stopwords-iso](https://github.com/stopwords-iso/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_iso_bn_3.4.1_3.0_1646673064513.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_iso_bn_3.4.1_3.0_1646673064513.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +------+ |result| +------+ |[ভাল] | +------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_iso| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|bn| |Size:|3.1 KB| --- layout: model title: English T5ForConditionalGeneration Tiny Cased model (from google) author: John Snow Labs name: t5_efficient_tiny_nl16 date: 2023-01-31 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-tiny-nl16` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl16_en_4.3.0_3.0_1675123795068.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_tiny_nl16_en_4.3.0_3.0_1675123795068.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_tiny_nl16","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_tiny_nl16","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_tiny_nl16| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|88.6 MB| ## References - https://huggingface.co/google/t5-efficient-tiny-nl16 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Lemmatizer (Romania, SpacyLookup) author: John Snow Labs name: lemma_spacylookup date: 2022-03-03 tags: [open_source, lemmatizer, ro] task: Lemmatization language: ro edition: Spark NLP 3.4.1 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This Romania Lemmatizer is an scalable, production-ready version of the Rule-based Lemmatizer available in [Spacy Lookups Data repository](https://github.com/explosion/spacy-lookups-data/). {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_ro_3.4.1_3.0_1646316527511.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/lemma_spacylookup_ro_3.4.1_3.0_1646316527511.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------------------------------+ |result | +-------------------------------+ |[Nu, fi, mai, bun, decât, mină]| +-------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|lemma_spacylookup| |Compatibility:|Spark NLP 3.4.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[lemma]| |Language:|ro| |Size:|3.4 MB| --- layout: model title: English asr_wav2vec2_base_timit_demo_colab_by_gullenasatish TFWav2Vec2ForCTC from gullenasatish author: John Snow Labs name: pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish date: 2022-09-24 tags: [wav2vec2, en, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_wav2vec2_base_timit_demo_colab_by_gullenasatish` is a English model originally trained by gullenasatish. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish_en_4.2.0_3.0_1664038844889.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish_en_4.2.0_3.0_1664038844889.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish', lang = 'en') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish", lang = "en") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_wav2vec2_base_timit_demo_colab_by_gullenasatish| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|en| |Size:|354.9 MB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: Pipeline to Detect chemicals in text author: John Snow Labs name: ner_chemicals_pipeline date: 2022-03-21 tags: [licensed, ner, clinical, en] task: Named Entity Recognition language: en nav_key: models edition: Healthcare NLP 3.4.1 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained pipeline is built on the top of [ner_chemicals](https://nlp.johnsnowlabs.com/2021/04/01/ner_chemicals_en.html) model. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/ner_chemicals_pipeline_en_3.4.1_3.0_1647869797628.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/ner_chemicals_pipeline_en_3.4.1_3.0_1647869797628.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline("ner_chemicals_pipeline", "en", "clinical/models") pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` ```scala val pipeline = new PretrainedPipeline("ner_chemicals_pipeline", "en", "clinical/models") pipeline.annotate("The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.") ``` {:.nlu-block} ```python import nlu nlu.load("en.med_ner.chemicals.pipeline").predict("""The results have shown that the product p - choloroaniline is not a significant factor in chlorhexidine - digluconate associated erosive cystitis. A high percentage of kanamycin - colistin and povidone - iodine irrigations were associated with erosive cystitis.""") ```

## Results ```bash +---------------------------+--------+ |chunks |entities| +---------------------------+--------+ |p - choloroaniline |CHEM | |chlorhexidine - digluconate|CHEM | |kanamycin |CHEM | |colistin |CHEM | |povidone - iodine |CHEM | +---------------------------+--------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|ner_chemicals_pipeline| |Type:|pipeline| |Compatibility:|Healthcare NLP 3.4.1+| |License:|Licensed| |Edition:|Official| |Language:|en| |Size:|1.7 GB| ## Included Models - DocumentAssembler - SentenceDetectorDLModel - TokenizerModel - WordEmbeddingsModel - MedicalNerModel - NerConverter --- layout: model title: Italian Bert Embeddings (Uncased) author: John Snow Labs name: bert_embeddings_bert_base_italian_xxl_uncased date: 2022-04-11 tags: [bert, embeddings, it, open_source] task: Embeddings language: it edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `bert-base-italian-xxl-uncased` is a Italian model orginally trained by `dbmdz`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_italian_xxl_uncased_it_3.4.2_3.0_1649676784263.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_bert_base_italian_xxl_uncased_it_3.4.2_3.0_1649676784263.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_bert_base_italian_xxl_uncased| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|it| |Size:|415.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/dbmdz/bert-base-italian-xxl-uncased - http://opus.nlpl.eu/ - https://traces1.inria.fr/oscar/ - https://github.com/dbmdz/berts/issues/7 - https://github.com/stefan-it/turkish-bert/tree/master/electra - https://github.com/stefan-it/italian-bertelectra - https://github.com/dbmdz/berts/issues/new --- layout: model title: German BertForMaskedLM Base Cased model author: John Snow Labs name: bert_embeddings_base_german_cased date: 2022-12-02 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-german-cased` is a German model originally trained by HuggingFace. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_german_cased_de_4.2.4_3.0_1670017649516.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_german_cased_de_4.2.4_3.0_1670017649516.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_german_cased","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_german_cased","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_german_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|409.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/bert-base-german-cased - https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png - https://github.com/deepset-ai/FARM/issues/60 - https://deepset.ai/german-bert - https://thumb.tildacdn.com/tild3162-6462-4566-b663-376630376138/-/format/webp/Screenshot_from_2020.png - https://thumb.tildacdn.com/tild6335-3531-4137-b533-313365663435/-/format/webp/deepset_checkpoints.png - https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png - https://deepset.ai/german-bert - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://deepset.ai --- layout: model title: Arabic Bert Embeddings (from Ebtihal) author: John Snow Labs name: bert_embeddings_AraBertMo_base_V1 date: 2022-04-11 tags: [bert, embeddings, ar, open_source] task: Embeddings language: ar edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `AraBertMo_base_V1` is a Arabic model orginally trained by `Ebtihal`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_AraBertMo_base_V1_ar_3.4.2_3.0_1649678935180.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_AraBertMo_base_V1_ar_3.4.2_3.0_1649678935180.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_AraBertMo_base_V1| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ar| |Size:|410.7 MB| |Case sensitive:|true| ## References - https://huggingface.co/Ebtihal/AraBertMo_base_V1 - https://github.com/google-research/bert - https://traces1.inria.fr/oscar/ - https://uokufa.edu.iq/ - https://mathcomp.uokufa.edu.iq/ --- layout: model title: Icelandic DistilBertForTokenClassification Cased model (from m3hrdadfi) author: John Snow Labs name: distilbert_token_classifier_icelandic_ner date: 2023-03-06 tags: [is, open_source, distilbert, token_classification, ner, tensorflow] task: Named Entity Recognition language: is edition: Spark NLP 4.3.1 spark_version: 3.0 supported: true engine: tensorflow annotator: DistilBertForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `icelandic-ner-distilbert` is a Icelandic model originally trained by `m3hrdadfi`. ## Predicted Entities `Money`, `Date`, `Time`, `Percent`, `Miscellaneous`, `Location`, `Person`, `Organization` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_icelandic_ner_distilbert_is_4.3.1_3.0_1678134372960.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_token_classifier_icelandic_ner_distilbert_is_4.3.1_3.0_1678134372960.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_icelandic_ner_distilbert","is") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") pipeline = Pipeline(stages=[documentAssembler, tokenizer, tokenClassifier]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_icelandic_ner_distilbert","is") .setInputCols(Array("document", "token")) .setOutputCol("ner") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, tokenClassifier)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_token_classifier_icelandic_ner_distilbert| |Compatibility:|Spark NLP 4.3.1+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|is| |Size:|505.8 MB| |Case sensitive:|true| |Max sentence length:|128| ## References - https://huggingface.co/m3hrdadfi/icelandic-ner-distilbert - http://hdl.handle.net/20.500.12537/42 - https://en.ru.is/ - https://github.com/m3hrdadfi/icelandic-ner/issues --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from mvonwyl) author: John Snow Labs name: distilbert_qa_mvonwyl_base_uncased_finetuned_squad2 date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad2` is a English model originally trained by `mvonwyl`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_mvonwyl_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773565472.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_mvonwyl_base_uncased_finetuned_squad2_en_4.3.0_3.0_1672773565472.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mvonwyl_base_uncased_finetuned_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_mvonwyl_base_uncased_finetuned_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_mvonwyl_base_uncased_finetuned_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/mvonwyl/distilbert-base-uncased-finetuned-squad2 --- layout: model title: English XLMRobertaForTokenClassification Large Cased model (from asahi417) author: John Snow Labs name: xlmroberta_ner_tner_large_bc5cdr date: 2022-08-13 tags: [en, open_source, xlm_roberta, ner] task: Named Entity Recognition language: en nav_key: models edition: Spark NLP 4.1.0 spark_version: 3.0 supported: true annotator: XlmRoBertaForTokenClassification article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained XLMRobertaForTokenClassification model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `tner-xlm-roberta-large-bc5cdr` is a English model originally trained by `asahi417`. ## Predicted Entities `chemical`, `disease` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_bc5cdr_en_4.1.0_3.0_1660424448383.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/xlmroberta_ner_tner_large_bc5cdr_en_4.1.0_3.0_1660424448383.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_bc5cdr","en") \ .setInputCols(["document", "token"]) \ .setOutputCol("ner") ner_converter = NerConverter()\ .setInputCols(["document", "token", "ner"])\ .setOutputCol("ner_chunk") pipeline = Pipeline(stages=[documentAssembler, tokenizer, token_classifier, ner_converter]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val token_classifier = XlmRoBertaForTokenClassification.pretrained("xlmroberta_ner_tner_large_bc5cdr","en") .setInputCols(Array("document", "token")) .setOutputCol("ner") val ner_converter = new NerConverter() .setInputCols(Array("document", "token', "ner")) .setOutputCol("ner_chunk") val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, token_classifier, ner_converter)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|xlmroberta_ner_tner_large_bc5cdr| |Compatibility:|Spark NLP 4.1.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[ner]| |Language:|en| |Size:|1.7 GB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/asahi417/tner-xlm-roberta-large-bc5cdr - https://github.com/asahi417/tner --- layout: model title: Sentence Entity Resolver for CPT codes (procedures and measurements) - Augmented author: John Snow Labs name: sbiobertresolve_cpt_procedures_measurements_augmented date: 2022-05-10 tags: [licensed, en, clinical, entity_resolution, cpt] task: Entity Resolution language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true recommended: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model maps medical entities to CPT codes using Sentence Bert Embeddings. The corpus of this model has been extented to measurements, and this model is capable of mapping both procedures and measurement concepts/entities to CPT codes. Measurement codes are helpful in codifying medical entities related to tests and their results. ## Predicted Entities `CPT Codes` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/healthcare/ER_CPT/){:.button.button-orange} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/ER_CPT.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_measurements_augmented_en_3.5.1_3.0_1652168576968.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/sbiobertresolve_cpt_procedures_measurements_augmented_en_3.5.1_3.0_1652168576968.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +---------------------+---------+--------+----------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+ | chunk| entity|cpt_code| all_k_resolutions| all_k_codes| +---------------------+---------+--------+----------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+ | CT scan of the chest| Test| 71250|Diagnostic CT scan of chest [Computed tomography, thorax, diagnostic; without contrast material]:...|71250:::70490:::76497:::71260:::74150:::70486:::73200:::70480:::77014:::73700:::71270:::70491:::7...| | pericardectomy|Procedure| 33030|Pericardectomy [Pericardiectomy, subtotal or complete; without cardiopulmonary bypass]:::Pericard...|33030:::33020:::64746:::49250:::27350:::68520:::32310:::27340:::33025:::32215:::41821:::1005708::...| | chest tube placement|Procedure| 39503|Insertion of chest tube [Repair, neonatal diaphragmatic hernia, with or without chest tube insert...|39503:::96440:::32553:::35820:::32100:::36226:::21899:::29200:::0174T:::31502:::31605:::69424:::1...| |drainage of the fluid|Procedure| 10140|Drainage of blood or fluid accumulation [Incision and drainage of hematoma, seroma or fluid colle...|10140:::40800:::61108:::41006:::62180:::83986:::49082:::27030:::21502:::49323:::32554:::51040:::6...| | thoracoscopy|Procedure| 1020900|Thoracoscopy [Thoracoscopy]:::Thoracoscopy, surgical; with control of traumatic hemorrhage | [Hea...| 1020900:::32654:::32668:::1006014:::35820:::32606:::32555:::31781:::31515:::29200| +---------------------+---------+--------+----------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+ ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|sbiobertresolve_cpt_procedures_measurements_augmented| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[cpt_code]| |Language:|en| |Size:|360.4 MB| |Case sensitive:|false| ## References Trained on Current Procedural Terminology dataset with `sbiobert_base_cased_mli` sentence embeddings. --- layout: model title: English DistilBertForQuestionAnswering Base Uncased model (from Plimpton) author: John Snow Labs name: distilbert_qa_plimpton_base_uncased_finetuned_squad date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Plimpton`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_plimpton_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768921810.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_plimpton_base_uncased_finetuned_squad_en_4.3.0_3.0_1672768921810.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_plimpton_base_uncased_finetuned_squad","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_plimpton_base_uncased_finetuned_squad","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_plimpton_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Plimpton/distilbert-base-uncased-finetuned-squad --- layout: model title: Legal Indenture Document Binary Classifier (Longformer) author: John Snow Labs name: legclf_indenture_agreement date: 2022-12-18 tags: [en, legal, classification, licensed, document, longformer, indenture, tensorflow] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description The `legclf_indenture_agreement` model is a Longformer Document Classifier used to classify if the document belongs to the class `indenture` or not (Binary Classification). Longformers have a restriction on 4096 tokens, so only the first 4096 tokens will be taken into account. We have realised that for the big majority of the documents in legal corpora, if they are clean and only contain the legal document without any extra information before, 4096 is enough to perform Document Classification. If your document needs to process more than 4096 tokens, you can try the following: getting chunks of 4096 tokens and average the embeddings, training with the averaged version, what means all document will be taken into account. ## Predicted Entities `indenture`, `other` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_indenture_agreement_en_1.0.0_3.0_1671393672864.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_indenture_agreement_en_1.0.0_3.0_1671393672864.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

## Results ```bash +-------+ |result| +-------+ |[indenture]| |[other]| |[other]| |[indenture]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_indenture_agreement| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[class]| |Language:|en| |Size:|21.8 MB| ## References Legal documents, scrapped from the Internet, and classified in-house + SEC documents ## Benchmarking ```bash label precision recall f1-score support indenture 0.90 0.94 0.92 113 other 0.96 0.94 0.95 188 accuracy - - 0.94 301 macro-avg 0.93 0.94 0.93 301 weighted-avg 0.94 0.94 0.94 301 ``` --- layout: model title: Stop Words Cleaner for French author: John Snow Labs name: stopwords_fr date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: fr edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, fr] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_fr_fr_2.5.4_2.4_1594742439495.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_fr_fr_2.5.4_2.4_1594742439495.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_fr", "fr") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("En plus d'être le roi du nord, John Snow est un médecin anglais et un leader dans le développement de l'anesthésie et de l'hygiène médicale.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_fr", "fr") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("En plus d"être le roi du nord, John Snow est un médecin anglais et un leader dans le développement de l"anesthésie et de l'hygiène médicale.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""En plus d'être le roi du nord, John Snow est un médecin anglais et un leader dans le développement de l'anesthésie et de l'hygiène médicale."""] stopword_df = nlu.load('fr.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=8, end=13, result="d'être", metadata={'sentence': '0'}), Row(annotatorType='token', begin=18, end=20, result='roi', metadata={'sentence': '0'}), Row(annotatorType='token', begin=25, end=28, result='nord', metadata={'sentence': '0'}), Row(annotatorType='token', begin=29, end=29, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=31, end=34, result='John', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_fr| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|fr| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords) --- layout: model title: Translate English to Tahitian Pipeline author: John Snow Labs name: translate_en_ty date: 2021-01-03 task: [Translation, Pipeline Public] language: xx edition: Spark NLP 2.7.0 spark_version: 2.4 tags: [open_source, seq2seq, translation, pipeline, en, ty, xx] supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Marian is an efficient, free Neural Machine Translation framework written in pure C++ with minimal dependencies. It is mainly being developed by the Microsoft Translator team. Many academic (most notably the University of Edinburgh and in the past the Adam Mickiewicz University in Poznań) and commercial contributors help with its development. It is currently the engine behind the Microsoft Translator Neural Machine Translation services and being deployed by many companies, organizations and research projects (see below for an incomplete list). Note that this is a very computationally expensive module especially on larger sequence. The use of an accelerator such as GPU is recommended. - source languages: `en` - target languages: `ty` {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/translate_en_ty_xx_2.7.0_2.4_1609687934373.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/translate_en_ty_xx_2.7.0_2.4_1609687934373.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python from sparknlp.pretrained import PretrainedPipeline pipeline = PretrainedPipeline("translate_en_ty", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` ```scala import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline val pipeline = new PretrainedPipeline("translate_en_ty", lang = "xx") pipeline.annotate("Your sentence to translate!") ``` {:.nlu-block} ```python import nlu text = ["text to translate"] translate_df = nlu.load('xx.en.translate_to.ty').predict(text, output_level='sentence') translate_df ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|translate_en_ty| |Type:|pipeline| |Compatibility:|Spark NLP 2.7.0+| |Edition:|Official| |Language:|xx| ## Data Source [https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models](https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models) --- layout: model title: Legal Severability Clause Binary Classifier author: John Snow Labs name: legclf_severability_clause date: 2022-08-10 tags: [en, legal, classification, clauses, licensed] task: Text Classification language: en nav_key: models edition: Legal NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model is a Binary Classifier (True, False) for the `severability` clause type. To use this model, make sure you provide enough context as an input. Adding Sentence Splitters to the pipeline will make the model see only sentences, not the whole text, so it's better to skip it, unless you want to do Binary Classification as sentence level. If you have big legal documents, and you want to look for clauses, we recommend you to split the documents using any of the techniques available in our Legal NLP Workshop Tokenization & Splitting Tutorial (link [here](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings_JSL/Legal/1.Tokenization_Splitting.ipynb)), namely: - Paragraph splitting (by multiline); - Splitting by headers / subheaders; - etc. Take into consideration the embeddings of this model allows up to 512 tokens. If you have more than that, consider splitting in smaller pieces (you can also check the same tutorial link provided above). This model can be combined with any of the other "hundreds" of Legal Clauses Classifiers you will find in Models Hub, getting as an output a series of True/False values for each of the legal clause model you have added. ## Predicted Entities `other`, `severability` {:.btn-box} [Live Demo](https://demo.johnsnowlabs.com/finance/CLASSIFY_LEGAL_CLAUSES/){:.button.button-orange} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/legal/models/legclf_severability_clause_en_1.0.0_3.2_1660123999498.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/legal/models/legclf_severability_clause_en_1.0.0_3.2_1660123999498.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler() \ .setInputCol("clause_text") \ .setOutputCol("document") embeddings = nlp.BertSentenceEmbeddings.pretrained("sent_bert_base_cased", "en") \ .setInputCols("document") \ .setOutputCol("sentence_embeddings") docClassifier = nlp.ClassifierDLModel.pretrained("legclf_severability_clause", "en", "legal/models")\ .setInputCols(["sentence_embeddings"])\ .setOutputCol("category") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, embeddings, docClassifier]) df = spark.createDataFrame([["YOUR TEXT HERE"]]).toDF("clause_text") model = nlpPipeline.fit(df) result = model.transform(df) ```

## Results ```bash +-------+ | result| +-------+ |[severability]| |[other]| |[other]| |[severability]| ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|legclf_severability_clause| |Compatibility:|Legal NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[sentence_embeddings]| |Output Labels:|[category]| |Language:|en| |Size:|23.0 MB| ## References Legal documents, scrapped from the Internet, and classified in-house ## Benchmarking ```bash label precision recall f1-score support other 0.97 0.98 0.98 131 severability 0.97 0.95 0.96 81 accuracy - - 0.97 212 macro-avg 0.97 0.97 0.97 212 weighted-avg 0.97 0.97 0.97 212 ``` --- layout: model title: Map Company Tickers to Subsidiary Companies (wikipedia, en) author: John Snow Labs name: finmapper_wikipedia_parentcompanies_ticker date: 2023-01-13 tags: [subsidiaries, companies, acquisitions, en, licensed] task: Chunk Mapping language: en nav_key: models edition: Finance NLP 1.0.0 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This models allows you to, given an extracter TICKER, retrieve all the parent / subsidiary /companies acquired and/or in the same group than it. ## Predicted Entities {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/finance/models/finmapper_wikipedia_parentcompanies_ticker_en_1.0.0_3.0_1673610848433.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/finance/models/finmapper_wikipedia_parentcompanies_ticker_en_1.0.0_3.0_1673610848433.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = nlp.DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentenceDetector = nlp.SentenceDetector()\ .setInputCols(["document"])\ .setOutputCol("sentence") tokenizer = nlp.Tokenizer()\ .setInputCols(["sentence"])\ .setOutputCol("token") embeddings = nlp.BertEmbeddings.pretrained("bert_embeddings_sec_bert_base","en") \ .setInputCols(["sentence", "token"]) \ .setOutputCol("embeddings") ner_model = finance.NerModel.pretrained("finner_ticker", "en", "finance/models")\ .setInputCols(["sentence", "token", "embeddings"])\ .setOutputCol("ner")\ ner_converter = nlp.NerConverter()\ .setInputCols(["sentence", "token", "ner"])\ .setOutputCol("ner_chunk") CM = finance.ChunkMapperModel()\ .pretrained('finmapper_wikipedia_parentcompanies_ticker','en','finance/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("mappings") nlpPipeline = nlp.Pipeline(stages=[ documentAssembler, sentenceDetector, tokenizer, embeddings, ner_model, ner_converter, CM ]) text = ["""ABG is a multinational corporation that is engaged in ..."""] test_data = spark.createDataFrame([text]).toDF("text") model = nlpPipeline.fit(test_data) lp = nlp.LightPipeline(model) res= model.transform(test_data) ```

## Results ```bash {'mappings': ['ABSA Group Limited', 'ABSA Group Limited@http://www.wikidata.org/entity/Q58641733', 'ABSA Group Limited@ABSA Group Limited@en', 'ABSA Group Limited@http://www.wikidata.org/prop/direct/P749', 'ABSA Group Limited@is_parent_of', 'ABSA Group Limited@JOHANNESBURG STOCK EXCHANGE@en', 'ABSA Group Limited@باركليز@ar', 'ABSA Group Limited@http://www.wikidata.org/entity/Q245343'], ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|finmapper_wikipedia_parentcompanies_ticker| |Compatibility:|Finance NLP 1.0.0+| |License:|Licensed| |Edition:|Official| |Input Labels:|[ner_chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|1.3 MB| ## References Wikidata --- layout: model title: Mapping RxNorm Codes with Corresponding National Drug Codes author: John Snow Labs name: rxnorm_ndc_mapper date: 2022-05-09 tags: [chunk_mapper, ndc, rxnorm, licensed, en, clinical] task: Chunk Mapping language: en nav_key: models edition: Healthcare NLP 3.5.1 spark_version: 3.0 supported: true annotator: ChunkMapperModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This pretrained model maps RxNorm and RxNorm Extension codes with corresponding National Drug Codes (NDC). ## Predicted Entities `Product NDC`, `Package NDC` {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/26.Chunk_Mapping.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapper_en_3.5.1_3.0_1652076748381.zip){:.button.button-orange.button-orange-trans.arr.button-icon.hidden} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/clinical/models/rxnorm_ndc_mapper_en_3.5.1_3.0_1652076748381.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = DocumentAssembler()\ .setInputCol('text')\ .setOutputCol('ner_chunk') sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\ .setInputCols(["ner_chunk"])\ .setOutputCol("sentence_embeddings")\ .setCaseSensitive(False) rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") \ .setInputCols(["sentence_embeddings"]) \ .setOutputCol("rxnorm_code")\ .setDistanceFunction("EUCLIDEAN") chunkerMapper_product = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("Product NDC")\ .setRel("Product NDC") chunkerMapper_package = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models")\ .setInputCols(["rxnorm_code"])\ .setOutputCol("Package NDC")\ .setRel("Package NDC") pipeline = Pipeline().setStages([document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_product, chunkerMapper_package ]) model = pipeline.fit(spark.createDataFrame([['']]).toDF('text')) lp = LightPipeline(model) result = lp.annotate(['doxepin hydrochloride 50 MG/ML', 'macadamia nut 100 MG/ML']) ``` ```scala val document_assembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("ner_chunk") val sbert_embedder = BertSentenceEmbeddings.pretrained("sbiobert_base_cased_mli", "en","clinical/models") .setInputCols(Array("ner_chunk")) .setOutputCol("sentence_embeddings") .setCaseSensitive(False) val rxnorm_resolver = SentenceEntityResolverModel.pretrained("sbiobertresolve_rxnorm_augmented","en", "clinical/models") .setInputCols(Array("sentence_embeddings")) .setOutputCol("rxnorm_code") .setDistanceFunction("EUCLIDEAN") val chunkerMapper_product = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models") .setInputCols(Array("rxnorm_code")) .setOutputCol("Product NDC") .setRel("Product NDC") val chunkerMapper_package = ChunkMapperModel.pretrained("rxnorm_ndc_mapper", "en", "clinical/models") .setInputCols(Array("rxnorm_code")) .setOutputCol("Package NDC") .setRel("Package NDC") val pipeline = new Pipeline().setStages(Array( document_assembler, sbert_embedder, rxnorm_resolver, chunkerMapper_product, chunkerMapper_package )) val text_data = Seq("doxepin hydrochloride 50 MG/ML", "macadamia nut 100 MG/ML").toDS.toDF("text") val res = pipeline.fit(text_data).transform(text_data) ``` {:.nlu-block} ```python import nlu nlu.load("en.rxnorm_to_ndc").predict("""Product NDC""") ```

## Results ```bash | | ner_chunk | rxnorm_code | Package NDC | Product NDC | |---:|:-----------------------------------|:--------------|:------------------|:---------------| | 0 | ['doxepin hydrochloride 50 MG/ML'] | ['1000091'] | ['00378-8117-45'] | ['00378-8117'] | | 1 | ['macadamia nut 100 MG/ML'] | ['212433'] | ['00064-2120-08'] | ['00064-2120'] | ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|rxnorm_ndc_mapper| |Compatibility:|Healthcare NLP 3.5.1+| |License:|Licensed| |Edition:|Official| |Input Labels:|[chunk]| |Output Labels:|[mappings]| |Language:|en| |Size:|4.2 MB| --- layout: model title: Javanese DistilBertForMaskedLM Small Cased model (from w11wo) author: John Snow Labs name: distilbert_embeddings_javanese_small_imdb date: 2022-12-12 tags: [jv, open_source, distilbert_embeddings, distilbertformaskedlm] task: Embeddings language: jv edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `javanese-distilbert-small-imdb` is a Javanese model originally trained by `w11wo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_small_imdb_jv_4.2.4_3.0_1670864962624.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_embeddings_javanese_small_imdb_jv_4.2.4_3.0_1670864962624.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_javanese_small_imdb","jv") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(False) pipeline = Pipeline(stages=[documentAssembler, tokenizer, distilbert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val distilbert_loaded = DistilBertEmbeddings.pretrained("distilbert_embeddings_javanese_small_imdb","jv") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(false) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, distilbert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_embeddings_javanese_small_imdb| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|jv| |Size:|247.9 MB| |Case sensitive:|false| ## References - https://huggingface.co/w11wo/javanese-distilbert-small-imdb - https://arxiv.org/abs/1910.01108 - https://github.com/sgugger - https://w11wo.github.io/ --- layout: model title: English asr_vakyansh_wav2vec2_maithili_maim_50 TFWav2Vec2ForCTC from Harveenchadha author: John Snow Labs name: asr_vakyansh_wav2vec2_maithili_maim_50 date: 2022-09-25 tags: [wav2vec2, en, audio, open_source, asr] task: Automatic Speech Recognition language: en nav_key: models edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: Wav2Vec2ForCTC article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_vakyansh_wav2vec2_maithili_maim_50` is a English model originally trained by Harveenchadha. NOTE: This model only works on a CPU, if you need to use this model on a GPU device please use asr_vakyansh_wav2vec2_maithili_maim_50_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/asr_vakyansh_wav2vec2_maithili_maim_50_en_4.2.0_3.0_1664117277718.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/asr_vakyansh_wav2vec2_maithili_maim_50_en_4.2.0_3.0_1664117277718.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python audio_assembler = AudioAssembler() \ .setInputCol("audio_content") \ .setOutputCol("audio_assembler") speech_to_text = Wav2Vec2ForCTC \ .pretrained("asr_vakyansh_wav2vec2_maithili_maim_50", "en")\ .setInputCols("audio_assembler") \ .setOutputCol("text") pipeline = Pipeline(stages=[ audio_assembler, speech_to_text, ]) pipelineModel = pipeline.fit(audioDf) pipelineDF = pipelineModel.transform(audioDf) ``` ```scala val audioAssembler = new AudioAssembler() .setInputCol("audio_content") .setOutputCol("audio_assembler") val speechToText = Wav2Vec2ForCTC .pretrained("asr_vakyansh_wav2vec2_maithili_maim_50", "en") .setInputCols("audio_assembler") .setOutputCol("text") val pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText)) val pipelineModel = pipeline.fit(audioDf) val pipelineDF = pipelineModel.transform(audioDf) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|asr_vakyansh_wav2vec2_maithili_maim_50| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[audio_assembler]| |Output Labels:|[text]| |Language:|en| |Size:|227.8 MB| --- layout: model title: German BertForMaskedLM Base Cased model (from deepset) author: John Snow Labs name: bert_embeddings_g_base date: 2022-12-06 tags: [de, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: de edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `gbert-base` is a German model originally trained by `deepset`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_base_de_4.2.4_3.0_1670326376987.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_g_base_de_4.2.4_3.0_1670326376987.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_base","de") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_g_base","de") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_g_base| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|de| |Size:|412.5 MB| |Case sensitive:|true| ## References - https://huggingface.co/deepset/gbert-base - https://arxiv.org/pdf/2010.10906.pdf - https://arxiv.org/pdf/2010.10906.pdf - https://workablehr.s3.amazonaws.com/uploads/account/logo/476306/logo - https://deepset.ai/german-bert - https://deepset.ai/germanquad - https://github.com/deepset-ai/FARM - https://github.com/deepset-ai/haystack/ - https://twitter.com/deepset_ai - https://www.linkedin.com/company/deepset-ai/ - https://haystack.deepset.ai/community/join - https://github.com/deepset-ai/haystack/discussions - https://deepset.ai - http://www.deepset.ai/jobs --- layout: model title: Turkish ElectraForQuestionAnswering model (from kuzgunlar) author: John Snow Labs name: electra_qa_turkish date: 2022-06-22 tags: [tr, open_source, electra, question_answering] task: Question Answering language: tr edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `electra-turkish-qa` is a Turkish model originally trained by `kuzgunlar`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/electra_qa_turkish_tr_4.0.0_3.0_1655921428307.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/electra_qa_turkish_tr_4.0.0_3.0_1655921428307.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("electra_qa_turkish","tr") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("electra_qa_turkish","tr") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Benim adım ne?", "Benim adım Clara ve Berkeley'de yaşıyorum.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("tr.answer_question.electra").predict("""Benim adım ne?|||"Benim adım Clara ve Berkeley'de yaşıyorum.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|electra_qa_turkish| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|tr| |Size:|412.7 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/kuzgunlar/electra-turkish-qa --- layout: model title: English RobertaForQuestionAnswering (from csarron) author: John Snow Labs name: roberta_qa_roberta_large_squad_v1 date: 2022-06-20 tags: [en, open_source, question_answering, roberta] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-large-squad-v1` is a English model originally trained by `csarron`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad_v1_en_4.0.0_3.0_1655737362298.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_roberta_large_squad_v1_en_4.0.0_3.0_1655737362298.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python document_assembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = RoBertaForQuestionAnswering.pretrained("roberta_qa_roberta_large_squad_v1","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer") \ .setCaseSensitive(True) pipeline = Pipeline().setStages([ document_assembler, spanClassifier ]) example = spark.createDataFrame([["What's my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(example).transform(example) ``` ```scala val document = new MultiDocumentAssembler() .setInputCols("question", "context") .setOutputCols("document_question", "document_context") val spanClassifier = RoBertaForQuestionAnswering .pretrained("roberta_qa_roberta_large_squad_v1","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) .setMaxSentenceLength(512) val pipeline = new Pipeline().setStages(Array(document, spanClassifier)) val example = Seq( ("Where was John Lenon born?", "John Lenon was born in London and lived in Paris. My name is Sarah and I live in London."), ("What's my name?", "My name is Clara and I live in Berkeley.")) .toDF("question", "context") val result = pipeline.fit(example).transform(example) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.roberta.large").predict("""What's my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_roberta_large_squad_v1| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[question, context]| |Output Labels:|[answer]| |Language:|en| |Size:|1.3 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/csarron/roberta-large-squad-v1 --- layout: model title: Russian BertForQuestionAnswering Cased model (from ruselkomp) author: John Snow Labs name: bert_qa_sber_full_tes date: 2022-07-07 tags: [ru, open_source, bert, question_answering] task: Question Answering language: ru edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: BertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `sber-full-test` is a Russian model originally trained by `ruselkomp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_qa_sber_full_tes_ru_4.0.0_3.0_1657191384050.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_qa_sber_full_tes_ru_4.0.0_3.0_1657191384050.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = BertForQuestionAnswering.pretrained("bert_qa_sber_full_tes","ru") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["Как меня зовут?", "Меня зовут Клара, и я живу в Беркли."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = BertForQuestionAnswering.pretrained("bert_qa_sber_full_tes","ru") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("Как меня зовут?", "Меня зовут Клара, и я живу в Беркли.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_qa_sber_full_tes| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|ru| |Size:|1.6 GB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/ruselkomp/sber-full-test --- layout: model title: English RobertaForQuestionAnswering Base Cased model (from ModelTC) author: John Snow Labs name: roberta_qa_modeltc_base_squad2 date: 2023-01-20 tags: [en, open_source, roberta, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained RobertaForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `roberta-base-squad2` is a English model originally trained by `ModelTC`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_qa_modeltc_base_squad2_en_4.3.0_3.0_1674218961578.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_qa_modeltc_base_squad2_en_4.3.0_3.0_1674218961578.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_modeltc_base_squad2","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = RoBertaForQuestionAnswering.pretrained("roberta_qa_modeltc_base_squad2","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_qa_modeltc_base_squad2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document, token]| |Output Labels:|[class]| |Language:|en| |Size:|464.3 MB| |Case sensitive:|true| |Max sentence length:|256| ## References - https://huggingface.co/ModelTC/roberta-base-squad2 --- layout: model title: Polish BertForMaskedLM Base Cased model (from Geotrend) author: John Snow Labs name: bert_embeddings_base_pl_cased date: 2022-12-02 tags: [pl, open_source, bert_embeddings, bertformaskedlm] task: Embeddings language: pl edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained BertForMaskedLM model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `bert-base-pl-cased` is a Polish model originally trained by `Geotrend`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_pl_cased_pl_4.2.4_3.0_1670018669418.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_base_pl_cased_pl_4.2.4_3.0_1670018669418.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols(["text"]) \ .setOutputCols("document") tokenizer = Tokenizer() \ .setInputCols("document") \ .setOutputCol("token") bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_pl_cased","pl") \ .setInputCols(["document", "token"]) \ .setOutputCol("embeddings") \ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, tokenizer, bert_loaded]) data = spark.createDataFrame([["I love Spark NLP"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols(Array("text")) .setOutputCols(Array("document")) val tokenizer = new Tokenizer() .setInputCols("document") .setOutputCol("token") val bert_loaded = BertEmbeddings.pretrained("bert_embeddings_base_pl_cased","pl") .setInputCols(Array("document", "token")) .setOutputCol("embeddings") .setCaseSensitive(True) val pipeline = new Pipeline().setStages(Array(documentAssembler, tokenizer, bert_loaded)) val data = Seq("I love Spark NLP").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_base_pl_cased| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|pl| |Size:|387.6 MB| |Case sensitive:|true| ## References - https://huggingface.co/Geotrend/bert-base-pl-cased - https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf - https://github.com/Geotrend-research/smaller-transformers --- layout: model title: Lithuanian Legal Roberta Embeddings author: John Snow Labs name: roberta_base_lithuanian_legal date: 2023-02-16 tags: [lt, lithuanian, embeddings, transformer, open_source, legal, tensorflow] task: Embeddings language: lt edition: Spark NLP 4.2.4 spark_version: 3.0 supported: true engine: tensorflow annotator: RoBertaEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Legal Roberta Embeddings model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `legal-lithuanian-roberta-base` is a Lithuanian model originally trained by `joelito`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/roberta_base_lithuanian_legal_lt_4.2.4_3.0_1676555711899.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/roberta_base_lithuanian_legal_lt_4.2.4_3.0_1676555711899.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_lithuanian_legal", "lt")\ .setInputCols(["sentence"])\ .setOutputCol("embeddings") ``` ```scala val sentence_embeddings = RoBertaEmbeddings.pretrained("roberta_base_lithuanian_legal", "lt") .setInputCols("sentence") .setOutputCol("embeddings")) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|roberta_base_lithuanian_legal| |Compatibility:|Spark NLP 4.2.4+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[embeddings]| |Language:|lt| |Size:|416.0 MB| |Case sensitive:|true| ## References https://huggingface.co/joelito/legal-lithuanian-roberta-base --- layout: model title: English DistilBertForQuestionAnswering model (from Adrian) Squad1 author: John Snow Labs name: distilbert_qa_Adrian_base_uncased_finetuned_squad date: 2022-06-08 tags: [en, open_source, distilbert, question_answering] task: Question Answering language: en nav_key: models edition: Spark NLP 4.0.0 spark_version: 3.0 supported: true annotator: DistilBertForQuestionAnswering article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Question Answering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert-base-uncased-finetuned-squad` is a English model originally trained by `Adrian`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_Adrian_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724056294.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_Adrian_base_uncased_finetuned_squad_en_4.0.0_3.0_1654724056294.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = MultiDocumentAssembler() \ .setInputCols(["question", "context"]) \ .setOutputCols(["document_question", "document_context"]) spanClassifier = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Adrian_base_uncased_finetuned_squad","en") \ .setInputCols(["document_question", "document_context"]) \ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[documentAssembler, spanClassifier]) data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val spanClassifer = DistilBertForQuestionAnswering.pretrained("distilbert_qa_Adrian_base_uncased_finetuned_squad","en") .setInputCols(Array("document", "token")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(documentAssembler, spanClassifier)) val data = Seq("What is my name?", "My name is Clara and I live in Berkeley.").toDF("question", "context") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu nlu.load("en.answer_question.squad.distil_bert.base_uncased.by_Adrian").predict("""What is my name?|||"My name is Clara and I live in Berkeley.""") ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_Adrian_base_uncased_finetuned_squad| |Compatibility:|Spark NLP 4.0.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.5 MB| |Case sensitive:|false| |Max sentence length:|512| ## References - https://huggingface.co/Adrian/distilbert-base-uncased-finetuned-squad --- layout: model title: English T5ForConditionalGeneration Large Cased model (from google) author: John Snow Labs name: t5_efficient_large_dl2 date: 2023-01-30 tags: [en, open_source, t5, tensorflow] task: Text Generation language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow annotator: T5Transformer article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained T5ForConditionalGeneration model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `t5-efficient-large-dl2` is a English model originally trained by `google`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/t5_efficient_large_dl2_en_4.3.0_3.0_1675114295600.zip){:.button.button-orange} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/t5_efficient_large_dl2_en_4.3.0_3.0_1675114295600.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python documentAssembler = DocumentAssembler() \ .setInputCols("text") \ .setOutputCols("document") t5 = T5Transformer.pretrained("t5_efficient_large_dl2","en") \ .setInputCols("document") \ .setOutputCol("answers") pipeline = Pipeline(stages=[documentAssembler, t5]) data = spark.createDataFrame([["PUT YOUR STRING HERE"]]).toDF("text") result = pipeline.fit(data).transform(data) ``` ```scala val documentAssembler = new DocumentAssembler() .setInputCols("text") .setOutputCols("document") val t5 = T5Transformer.pretrained("t5_efficient_large_dl2","en") .setInputCols("document") .setOutputCol("answers") val pipeline = new Pipeline().setStages(Array(documentAssembler, t5)) val data = Seq("PUT YOUR STRING HERE").toDS.toDF("text") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|t5_efficient_large_dl2| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[documents]| |Output Labels:|[t5]| |Language:|en| |Size:|770.5 MB| ## References - https://huggingface.co/google/t5-efficient-large-dl2 - https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html - https://arxiv.org/abs/2109.10686 - https://arxiv.org/abs/2109.10686 - https://github.com/google-research/google-research/issues/986#issuecomment-1035051145 --- layout: model title: Korean Bert Embeddings (from snunlp) author: John Snow Labs name: bert_embeddings_KR_FinBert date: 2022-04-11 tags: [bert, embeddings, ko, open_source] task: Embeddings language: ko edition: Spark NLP 3.4.2 spark_version: 3.0 supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Bert Embeddings model, uploaded to Hugging Face, adapted and imported into Spark NLP. `KR-FinBert` is a Korean model orginally trained by `snunlp`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/bert_embeddings_KR_FinBert_ko_3.4.2_3.0_1649675550368.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/bert_embeddings_KR_FinBert_ko_3.4.2_3.0_1649675550368.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|bert_embeddings_KR_FinBert| |Compatibility:|Spark NLP 3.4.2+| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[bert]| |Language:|ko| |Size:|380.8 MB| |Case sensitive:|true| ## References - https://huggingface.co/snunlp/KR-FinBert - https://www.kaggle.com/junbumlee/kcbert-pretraining-corpus-korean-news-comments --- layout: model title: German asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412 TFWav2Vec2ForCTC from jonatasgrosman author: John Snow Labs name: pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412 date: 2022-09-25 tags: [wav2vec2, de, audio, open_source, pipeline, asr] task: Automatic Speech Recognition language: de edition: Spark NLP 4.2.0 spark_version: 3.0 supported: true annotator: PipelineModel article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained Wav2vec2 pipeline, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP.`asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412` is a German model originally trained by jonatasgrosman. NOTE: This pipeline only works on a CPU, if you need to use this pipeline on a GPU device please use pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_gpu {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_de_4.2.0_3.0_1664112298280.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412_de_4.2.0_3.0_1664112298280.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python pipeline = PretrainedPipeline('pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412', lang = 'de') annotations = pipeline.transform(audioDF) ``` ```scala val pipeline = new PretrainedPipeline("pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412", lang = "de") val annotations = pipeline.transform(audioDF) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|pipeline_asr_exp_w2v2r_xls_r_accent_germany_5_austria_5_s412| |Type:|pipeline| |Compatibility:|Spark NLP 4.2.0+| |License:|Open Source| |Edition:|Official| |Language:|de| |Size:|1.2 GB| ## Included Models - AudioAssembler - Wav2Vec2ForCTC --- layout: model title: English DistilBertForQuestionAnswering Cased model (from nlpunibo) author: John Snow Labs name: distilbert_qa_convolutional_classifier date: 2023-01-03 tags: [en, open_source, distilbert, question_answering, tensorflow] task: Question Answering language: en nav_key: models edition: Spark NLP 4.3.0 spark_version: 3.0 supported: true engine: tensorflow article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description Pretrained DistilBertForQuestionAnswering model, adapted from Hugging Face and curated to provide scalability and production-readiness using Spark NLP. `distilbert_convolutional_classifier` is a English model originally trained by `nlpunibo`. {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/distilbert_qa_convolutional_classifier_en_4.3.0_3.0_1672774548669.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/distilbert_qa_convolutional_classifier_en_4.3.0_3.0_1672774548669.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python Document_Assembler = MultiDocumentAssembler()\ .setInputCols(["question", "context"])\ .setOutputCols(["document_question", "document_context"]) Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_convolutional_classifier","en")\ .setInputCols(["document_question", "document_context"])\ .setOutputCol("answer")\ .setCaseSensitive(True) pipeline = Pipeline(stages=[Document_Assembler, Question_Answering]) data = spark.createDataFrame([["What's my name?","My name is Clara and I live in Berkeley."]]).toDF("question", "context") result = pipeline.fit(data).transform(data) ``` ```scala val Document_Assembler = new MultiDocumentAssembler() .setInputCols(Array("question", "context")) .setOutputCols(Array("document_question", "document_context")) val Question_Answering = DistilBertForQuestionAnswering.pretrained("distilbert_qa_convolutional_classifier","en") .setInputCols(Array("document_question", "document_context")) .setOutputCol("answer") .setCaseSensitive(true) val pipeline = new Pipeline().setStages(Array(Document_Assembler, Question_Answering)) val data = Seq("What's my name?","My name is Clara and I live in Berkeley.").toDS.toDF("question", "context") val result = pipeline.fit(data).transform(data) ```

{:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|distilbert_qa_convolutional_classifier| |Compatibility:|Spark NLP 4.3.0+| |License:|Open Source| |Edition:|Official| |Input Labels:|[document_question, document_context]| |Output Labels:|[answer]| |Language:|en| |Size:|247.6 MB| |Case sensitive:|true| |Max sentence length:|512| ## References - https://huggingface.co/nlpunibo/distilbert_convolutional_classifier --- layout: model title: BioBERT Embeddings (Discharge) author: John Snow Labs name: biobert_discharge_base_cased date: 2020-08-25 task: Embeddings language: en nav_key: models edition: Spark NLP 2.6.0 spark_version: 2.4 tags: [embeddings, en, open_source] supported: true annotator: BertEmbeddings article_header: type: cover use_language_switcher: "Python-Scala-Java" --- ## Description This model contains a pre-trained weights of ClinicalBERT for discharge summaries. This domain-specific model has performance improvements on 3/5 clinical NLP tasks andd establishing a new state-of-the-art on the MedNLI dataset. The details are described in the paper "[Publicly Available Clinical BERT Embeddings](https://www.aclweb.org/anthology/W19-1909/)". {:.btn-box} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/biobert_discharge_base_cased_en_2.6.0_2.4_1598343571130.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/biobert_discharge_base_cased_en_2.6.0_2.4_1598343571130.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} ## How to use

{:.h2_title} ## Results ```bash token en_embed_biobert_discharge_base_cased_embeddings I [0.0036486536264419556, 0.3796533942222595, -0... hate [0.1914958357810974, 0.6709488034248352, -0.49... cancer [0.04618441313505173, -0.04562612622976303, -0... ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|biobert_discharge_base_cased| |Type:|embeddings| |Compatibility:|Spark NLP 2.6.0| |License:|Open Source| |Edition:|Official| |Input Labels:|[sentence, token]| |Output Labels:|[word_embeddings]| |Language:|[en]| |Dimension:|768| |Case sensitive:|true| {:.h2_title} ## Data Source The model is imported from [https://github.com/EmilyAlsentzer/clinicalBERT](https://github.com/EmilyAlsentzer/clinicalBERT) --- layout: model title: Stop Words Cleaner for Czech author: John Snow Labs name: stopwords_cs date: 2020-07-14 19:03:00 +0800 task: Stop Words Removal language: cs edition: Spark NLP 2.5.4 spark_version: 2.4 tags: [stopwords, cs] supported: true annotator: StopWordsCleaner article_header: type: cover use_language_switcher: "Python-Scala-Java" --- {:.h2_title} ## Description This model removes 'stop words' from text. Stop words are words so common that they can be removed without significantly altering the meaning of a text. Removing stop words is useful when one wants to deal with only the most semantically important words in a text, and ignore words that are rarely semantically relevant, such as articles and prepositions. {:.btn-box} [Open in Colab](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/b2eb08610dd49d5b15077cc499a94b4ec1e8b861/jupyter/annotation/english/stop-words/StopWordsCleaner.ipynb){:.button.button-orange.button-orange-trans.co.button-icon} [Download](https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/models/stopwords_cs_cs_2.5.4_2.4_1594742440427.zip){:.button.button-orange.button-orange-trans.arr.button-icon} [Copy S3 URI](s3://auxdata.johnsnowlabs.com/public/models/stopwords_cs_cs_2.5.4_2.4_1594742440427.zip){:.button.button-orange.button-orange-trans.button-icon.button-copy-s3} {:.h2_title} ## How to use

{% include programmingLanguageSelectScalaPythonNLU.html %} ```python ... stop_words = StopWordsCleaner.pretrained("stopwords_cs", "cs") \ .setInputCols(["token"]) \ .setOutputCol("cleanTokens") nlp_pipeline = Pipeline(stages=[document_assembler, tokenizer, stop_words]) light_pipeline = LightPipeline(nlp_pipeline.fit(spark.createDataFrame([['']]).toDF("text"))) results = light_pipeline.fullAnnotate("Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny.") ``` ```scala ... val stopWords = StopWordsCleaner.pretrained("stopwords_cs", "cs") .setInputCols(Array("token")) .setOutputCol("cleanTokens") val pipeline = new Pipeline().setStages(Array(document_assembler, tokenizer, stopWords)) val data = Seq("Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny.").toDF("text") val result = pipeline.fit(data).transform(data) ``` {:.nlu-block} ```python import nlu text = ["""Kromě toho, že je králem severu, je John Snow anglickým lékařem a lídrem ve vývoji anestezie a lékařské hygieny."""] stopword_df = nlu.load('cs.stopwords').predict(text) stopword_df[["cleanTokens"]] ```

{:.h2_title} ## Results ```bash [Row(annotatorType='token', begin=0, end=4, result='Kromě', metadata={'sentence': '0'}), Row(annotatorType='token', begin=10, end=10, result=',', metadata={'sentence': '0'}), Row(annotatorType='token', begin=12, end=13, result='že', metadata={'sentence': '0'}), Row(annotatorType='token', begin=18, end=23, result='králem', metadata={'sentence': '0'}), Row(annotatorType='token', begin=25, end=30, result='severu', metadata={'sentence': '0'}), ...] ``` {:.model-param} ## Model Information {:.table-model} |---|---| |Model Name:|stopwords_cs| |Type:|stopwords| |Compatibility:|Spark NLP 2.5.4+| |Edition:|Official| |Input Labels:|[token]| |Output Labels:|[cleanTokens]| |Language:|cs| |Case sensitive:|false| |License:|Open Source| {:.h2_title} ## Data Source The model is imported from [https://github.com/WorldBrain/remove-stopwords](https://github.com/WorldBrain/remove-stopwords)